lexer: loop if the current character does not produce a token

lexer.cpp

@@ -143,6 +143,7 @@ std::optional<token_t> lexer_t::scan_token()
 {
   using enum token_t::type_t;
 
+  while (true) {
     if (at_end_p())
       return {{pos, eof, ""}};
 
@@ -178,8 +179,12 @@ std::optional<token_t> lexer_t::scan_token()
     case '\t':
       break;
     case '\n':
+      {
+	token_pos_t tmp = pos;
 	pos.line++;
 	pos.col = 0;
+	return {{tmp, eol, lexeme()}};
+      }
       break;
     case '$':
       if (hex_t::pred(peek())) {
@@ -206,6 +211,7 @@ std::optional<token_t> lexer_t::scan_token()
       }
       break;
     }
+  }
   __builtin_unreachable();
 }
 

parser.cpp

@@ -159,4 +159,15 @@ expr_t * parser_t::primary()
   throw error(peek(), "expected expression");
 }
 
+/*
+void parser_t::synchronize()
+{
+  advance();
+  while (!at_end_p()) {
+    if (previous().type == eol) return;
+    advance();
+  }
+}
+*/
+
 }

token.hpp

@@ -85,6 +85,7 @@ struct token_t {
     _ends,
 
     eof,
+    eol,
   };
 
   using literal_t = std::variant<std::monostate, num_type>;
@@ -172,6 +173,7 @@ struct token_t {
     case _ends        : return os << "ENDS";
 
     case eof          : return os << "EOF";
+    case eol          : return os << "EOL";
     }
     __builtin_unreachable();
   }