From 9fd0105a8e68e57d065f27929ba4091dbfbc0e4d Mon Sep 17 00:00:00 2001 From: Team Humaki Date: Tue, 15 Sep 2026 22:26:38 -0700 Subject: [PATCH] scanner: emit BAD-STRING and keep scanning Unclosed quotes are a , not a fatal Scanner.err. An unescaped newline ends the bad string and is reconsumed so later rules still tokenize. --- scanner/doc.go | 10 ++++--- scanner/scanner.go | 46 ++++++++++++++++++++++------- scanner/scanner_test.go | 65 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 15 deletions(-) diff --git a/scanner/doc.go b/scanner/doc.go index f19850e..824a0c5 100644 --- a/scanner/doc.go +++ b/scanner/doc.go @@ -21,10 +21,12 @@ the token returned has type TokenEOF or TokenError: // Do something with the token... } -Following the CSS3 specification, an error can only occur when the scanner -finds an unclosed quote or unclosed comment. In these cases the text becomes -"untokenizable". Everything else is tokenizable and it is up to a parser -to make sense of the token stream (or ignore nonsensical token sequences). +Following the CSS3 specification, an unclosed quote emits a BAD-STRING token +and scanning continues (an unescaped newline ends the bad string and is +reconsumed). An error still occurs for an unclosed comment, which makes the +rest of the input untokenizable. Everything else is tokenizable and it is up +to a parser to make sense of the token stream (or ignore nonsensical token +sequences). Note: the scanner doesn't perform lexical analysis or, in other words, it doesn't care about the token context. It is intended to be used by a diff --git a/scanner/scanner.go b/scanner/scanner.go index 25a7c65..5d9bb5b 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -67,6 +67,7 @@ const ( TokenSubstringMatch TokenChar TokenBOM + TokenBadString ) // tokenNames maps tokenType's to their names. Used for conversion to string. @@ -94,6 +95,7 @@ var tokenNames = map[tokenType]string{ TokenSubstringMatch: "SUBSTRINGMATCH", TokenChar: "CHAR", TokenBOM: "BOM", + TokenBadString: "BAD-STRING", } // Macros and productions ----------------------------------------------------- @@ -217,7 +219,8 @@ type Scanner struct { // At the end of the input the token type is TokenEOF. // // If the input can't be tokenized the token type is TokenError. This occurs -// in case of unclosed quotation marks or comments. +// in case of unclosed comments. Unclosed quotes emit TokenBadString and +// scanning continues. func (s *Scanner) Next() *Token { if s.err != nil { return s.err @@ -262,14 +265,9 @@ func (s *Scanner) Next() *Token { // More common chars. return s.emitSimple(TokenChar, string(input[0])) case '"', '\'': - // String or error. - match := matchers[TokenString].FindString(input) - if match != "" { - return s.emitToken(TokenString, match) - } - - s.err = &Token{TokenError, "unclosed quotation mark", s.row, s.col} - return s.err + // STRING, or BAD-STRING if the quotes never close (newline or EOF). + tok, v := scanQuoted(input) + return s.emitToken(tok, v) case '/': // Comment, error or Char. if len(input) > 1 && input[1] == '*' { @@ -307,8 +305,7 @@ func (s *Scanner) Next() *Token { return s.emitToken(token, match) } } - // We already handled unclosed quotation marks and comments, - // so this can only be a Char. + // We already handled quotes and unclosed comments, so this can only be a Char. r, width := utf8.DecodeRuneInString(input) token := &Token{TokenChar, string(r), s.row, s.col} s.col += width @@ -316,6 +313,33 @@ func (s *Scanner) Next() *Token { return token } +// scanQuoted consumes a CSS string starting at input[0] ('"' or '\''). +// +// A matching unescaped quote yields TokenString. An unescaped newline or EOF +// yields TokenBadString; the newline is reconsumed so scanning can continue. +func scanQuoted(input string) (tokenType, string) { + quote := input[0] + i := 1 + for i < len(input) { + switch input[i] { + case '\\': + i++ + if i < len(input) { + _, w := utf8.DecodeRuneInString(input[i:]) + i += w + } + case '\n': + return TokenBadString, input[:i] + default: + if input[i] == quote { + return TokenString, input[:i+1] + } + i++ + } + } + return TokenBadString, input[:i] +} + // updatePosition updates input coordinates based on the consumed text. func (s *Scanner) updatePosition(text string) { width := utf8.RuneCountInString(text) diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index c1ab243..8902e5a 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -74,6 +74,71 @@ func TestMatchers(t *testing.T) { checkMatch(`╯︵┻━┻"stuff"`, TokenIdent, "╯︵┻━┻", TokenString, `"stuff"`) } +func TestBadString(t *testing.T) { + checkMatch := func(s string, ttList ...interface{}) { + t.Helper() + scanner := New(s) + + i := 0 + for i < len(ttList) { + tt := ttList[i].(tokenType) + tVal := ttList[i+1].(string) + if tok := scanner.Next(); tok.Type != tt || tok.Value != tVal { + t.Errorf("%q: got %v, want %s %q", s, tok, tt, tVal) + } + i += 2 + } + + if tok := scanner.Next(); tok.Type != TokenEOF { + t.Errorf("%q: missing EOF, got %+v", s, tok) + } + // Scanner.err must not stay latched on a bad string. + if tok := scanner.Next(); tok.Type != TokenEOF { + t.Errorf("%q: Next after EOF got %+v", s, tok) + } + } + + // Apostrophe starts a string; emit BAD-STRING and keep going (issue #12). + checkMatch(`@mspfa title 56 56 Who's this d****e?; .foo{}`, + TokenAtKeyword, "@mspfa", + TokenS, " ", + TokenIdent, "title", + TokenS, " ", + TokenNumber, "56", + TokenS, " ", + TokenNumber, "56", + TokenS, " ", + TokenIdent, "Who", + TokenBadString, `'s this d****e?; .foo{}`, + ) + + // Unescaped newline ends the bad string; later rules still tokenize. + checkMatch("\"unclosed\n.foo{}", + TokenBadString, `"unclosed`, + TokenS, "\n", + TokenChar, ".", + TokenIdent, "foo", + TokenChar, "{", + TokenChar, "}", + ) + checkMatch("'unclosed\n.bar{color:red}", + TokenBadString, `'unclosed`, + TokenS, "\n", + TokenChar, ".", + TokenIdent, "bar", + TokenChar, "{", + TokenIdent, "color", + TokenChar, ":", + TokenIdent, "red", + TokenChar, "}", + ) + + // Unclosed quote at EOF. + checkMatch(`"unclosed`, TokenBadString, `"unclosed`) + checkMatch(`'unclosed`, TokenBadString, `'unclosed`) + checkMatch(`"`, TokenBadString, `"`) +} + func TestPreprocess(t *testing.T) { tcs := []struct{ desc, input, expected string }{ {