Use the lexer"s skip flag when skipping whitespace

koto-lang · Dec 12, 2023 · 716bf53 · 716bf53
1 parent e6de433
commit 716bf53
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 10 deletions.
diff --git a/src/scanner.c b/src/scanner.c
@@ -61,6 +61,10 @@ static inline void advance(TSLexer* lexer) {
   lexer->advance(lexer, false);
 }
 
+static inline void skip(TSLexer* lexer) {
+  lexer->advance(lexer, true);
+}
+
 typedef struct {
   uint32_t len;
   uint32_t cap;
@@ -115,7 +119,7 @@ static void skip_whitespace(TSLexer* lexer) {
       break;
     }
     printf("...skipping (%u)\n", next);
-    advance(lexer);
+    skip(lexer);
   }
 }
 
@@ -190,25 +194,22 @@ static void consume_multiline_comment(TSLexer* lexer) {
   }
 }
 
-static bool consume_comment(TSLexer* lexer) {
+static void consume_comment(TSLexer* lexer) {
   assert(lexer->lookahead == '#');
   advance(lexer);
 
   if (lexer->lookahead == '-') {
     advance(lexer);
     consume_multiline_comment(lexer);
   } else {
-    while (true) {
+    while (!lexer->eof(lexer)) {
       switch (lexer->lookahead) {
       case '\n':
-      case '\0':
-        return true;
+        return;
       }
       advance(lexer);
     }
   }
-
-  return false;
 }
 
 bool tree_sitter_koto_external_scanner_scan(
@@ -271,10 +272,10 @@ bool tree_sitter_koto_external_scanner_scan(
     while (true) {
       // new_indent = lexer->get_column(lexer);
       if (lexer->lookahead == '\r') {
-        advance(lexer);
+        skip(lexer);
       }
       if (lexer->lookahead == '\n') {
-        advance(lexer);
+        skip(lexer);
         newline = true;
       } else {
         break;

diff --git a/test/corpus/functions.txt b/test/corpus/functions.txt
@@ -46,7 +46,7 @@ function_block
 )
 
 ===========================================================================================
-function_unpacking
+function_arg_unpacking
 ===========================================================================================
 
 |a, b, ...| true