scanner: emit BAD-STRING and keep scanning

scanner/doc.go

@@ -21,10 +21,12 @@ the token returned has type TokenEOF or TokenError:
 		// Do something with the token...
 	}
 
-Following the CSS3 specification, an error can only occur when the scanner
-finds an unclosed quote or unclosed comment. In these cases the text becomes
-"untokenizable". Everything else is tokenizable and it is up to a parser
-to make sense of the token stream (or ignore nonsensical token sequences).
+Following the CSS3 specification, an unclosed quote emits a BAD-STRING token
+and scanning continues (an unescaped newline ends the bad string and is
+reconsumed). An error still occurs for an unclosed comment, which makes the
+rest of the input untokenizable. Everything else is tokenizable and it is up
+to a parser to make sense of the token stream (or ignore nonsensical token
+sequences).
 
 Note: the scanner doesn't perform lexical analysis or, in other words, it
 doesn't care about the token context. It is intended to be used by a

scanner/scanner.go

@@ -67,6 +67,7 @@ const (
 	TokenSubstringMatch
 	TokenChar
 	TokenBOM
+	TokenBadString
 )
 
 // tokenNames maps tokenType's to their names. Used for conversion to string.
@@ -94,6 +95,7 @@ var tokenNames = map[tokenType]string{
 	TokenSubstringMatch: "SUBSTRINGMATCH",
 	TokenChar:           "CHAR",
 	TokenBOM:            "BOM",
+	TokenBadString:      "BAD-STRING",
 }
 
 // Macros and productions -----------------------------------------------------
@@ -217,7 +219,8 @@ type Scanner struct {
 // At the end of the input the token type is TokenEOF.
 //
 // If the input can't be tokenized the token type is TokenError. This occurs
-// in case of unclosed quotation marks or comments.
+// in case of unclosed comments. Unclosed quotes emit TokenBadString and
+// scanning continues.
 func (s *Scanner) Next() *Token {
 	if s.err != nil {
 		return s.err
@@ -262,14 +265,9 @@ func (s *Scanner) Next() *Token {
 		// More common chars.
 		return s.emitSimple(TokenChar, string(input[0]))
 	case '"', '\'':
-		// String or error.
-		match := matchers[TokenString].FindString(input)
-		if match != "" {
-			return s.emitToken(TokenString, match)
-		}
-
-		s.err = &Token{TokenError, "unclosed quotation mark", s.row, s.col}
-		return s.err
+		// STRING, or BAD-STRING if the quotes never close (newline or EOF).
+		tok, v := scanQuoted(input)
+		return s.emitToken(tok, v)
 	case '/':
 		// Comment, error or Char.
 		if len(input) > 1 && input[1] == '*' {
@@ -307,15 +305,41 @@ func (s *Scanner) Next() *Token {
 			return s.emitToken(token, match)
 		}
 	}
-	// We already handled unclosed quotation marks and comments,
-	// so this can only be a Char.
+	// We already handled quotes and unclosed comments, so this can only be a Char.
 	r, width := utf8.DecodeRuneInString(input)
 	token := &Token{TokenChar, string(r), s.row, s.col}
 	s.col += width
 	s.pos += width
 	return token
 }
 
+// scanQuoted consumes a CSS string starting at input[0] ('"' or '\'').
+//
+// A matching unescaped quote yields TokenString. An unescaped newline or EOF
+// yields TokenBadString; the newline is reconsumed so scanning can continue.
+func scanQuoted(input string) (tokenType, string) {
+	quote := input[0]
+	i := 1
+	for i < len(input) {
+		switch input[i] {
+		case '\\':
+			i++
+			if i < len(input) {
+				_, w := utf8.DecodeRuneInString(input[i:])
+				i += w
+			}
+		case '\n':
+			return TokenBadString, input[:i]
+		default:
+			if input[i] == quote {
+				return TokenString, input[:i+1]
+			}
+			i++
+		}
+	}
+	return TokenBadString, input[:i]
+}
+
 // updatePosition updates input coordinates based on the consumed text.
 func (s *Scanner) updatePosition(text string) {
 	width := utf8.RuneCountInString(text)

scanner/scanner_test.go

@@ -74,6 +74,71 @@ func TestMatchers(t *testing.T) {
 	checkMatch(`╯︵┻━┻"stuff"`, TokenIdent, "╯︵┻━┻", TokenString, `"stuff"`)
 }
 
+func TestBadString(t *testing.T) {
+	checkMatch := func(s string, ttList ...interface{}) {
+		t.Helper()
+		scanner := New(s)
+
+		i := 0
+		for i < len(ttList) {
+			tt := ttList[i].(tokenType)
+			tVal := ttList[i+1].(string)
+			if tok := scanner.Next(); tok.Type != tt || tok.Value != tVal {
+				t.Errorf("%q: got %v, want %s %q", s, tok, tt, tVal)
+			}
+			i += 2
+		}
+
+		if tok := scanner.Next(); tok.Type != TokenEOF {
+			t.Errorf("%q: missing EOF, got %+v", s, tok)
+		}
+		// Scanner.err must not stay latched on a bad string.
+		if tok := scanner.Next(); tok.Type != TokenEOF {
+			t.Errorf("%q: Next after EOF got %+v", s, tok)
+		}
+	}
+
+	// Apostrophe starts a string; emit BAD-STRING and keep going (issue #12).
+	checkMatch(`@mspfa title 56 56 Who's this d****e?; .foo{}`,
+		TokenAtKeyword, "@mspfa",
+		TokenS, " ",
+		TokenIdent, "title",
+		TokenS, " ",
+		TokenNumber, "56",
+		TokenS, " ",
+		TokenNumber, "56",
+		TokenS, " ",
+		TokenIdent, "Who",
+		TokenBadString, `'s this d****e?; .foo{}`,
+	)
+
+	// Unescaped newline ends the bad string; later rules still tokenize.
+	checkMatch("\"unclosed\n.foo{}",
+		TokenBadString, `"unclosed`,
+		TokenS, "\n",
+		TokenChar, ".",
+		TokenIdent, "foo",
+		TokenChar, "{",
+		TokenChar, "}",
+	)
+	checkMatch("'unclosed\n.bar{color:red}",
+		TokenBadString, `'unclosed`,
+		TokenS, "\n",
+		TokenChar, ".",
+		TokenIdent, "bar",
+		TokenChar, "{",
+		TokenIdent, "color",
+		TokenChar, ":",
+		TokenIdent, "red",
+		TokenChar, "}",
+	)
+
+	// Unclosed quote at EOF.
+	checkMatch(`"unclosed`, TokenBadString, `"unclosed`)
+	checkMatch(`'unclosed`, TokenBadString, `'unclosed`)
+	checkMatch(`"`, TokenBadString, `"`)
+}
+
 func TestPreprocess(t *testing.T) {
 	tcs := []struct{ desc, input, expected string }{
 		{