Improve block parsing

koto-lang · Dec 12, 2023 · 896ced7 · 896ced7
1 parent 7b8b2a8
commit 896ced7
Show file tree

Hide file tree

Showing 6 changed files with 182 additions and 90 deletions.
diff --git a/grammar.js b/grammar.js
@@ -62,13 +62,14 @@ module.exports = grammar({
 
   rules: {
     module: $ => seq(
-      repeat($._newline),
+      $._block_start,
       repeat(
         seq(
-          $._expressions,
-          repeat($._newline),
+          $._block_continue,
+          $._block_expressions,
         )
       ),
+      $._block_end,
     ),
 
     _term: $ => choice(
@@ -129,6 +130,21 @@ module.exports = grammar({
       $.expressions,
     ),
 
+    _block_expressions: $ => choice(
+      $._expressions,
+      $._cascade_arm,
+    ),
+
+    // Arms of if/else if/else or try/catch/finally cascades
+    // Ideally these would be included in $.if and $.try, but they intefere with block
+    // continuation. Parsing them separately will do for now.
+    _cascade_arm: $ => choice(
+      $.else_if,
+      $.else,
+      $.catch,
+      $.finally,
+    ),
+
     // comma-separated expressions with flexible newline rules
     _contained_expressions: $ => seq(
       repeat($._newline),
@@ -151,7 +167,7 @@ module.exports = grammar({
       repeat(
         seq(
           repeat1($._block_continue),
-          $._expressions,
+          $._block_expressions,
         )
       ),
       $._block_end,
@@ -205,7 +221,6 @@ module.exports = grammar({
 
     assign: $ => binary_op($, '=', prec.right, PREC.assign),
 
-
     modify_assign: $ => choice(
       binary_op($, '-=', prec.right, PREC.assign),
       binary_op($, '+=', prec.right, PREC.assign),
@@ -396,24 +411,20 @@ module.exports = grammar({
         'if',
         field('condition', $._expression),
         field('then', $.block),
-        repeat(
-          field('else_if', seq(
-            $._block_continue,
-            'else if',
-            field('else_if_condition', $._expression),
-            field('else_if_then', $.block),
-          )),
-        ),
-        optional(
-          seq(
-            $._block_continue,
-            'else',
-            field('else', $.block),
-          )
-        ),
       )),
     ),
 
+    else_if: $ => seq(
+      'else if',
+      field('condition', $._expression),
+      field('then', $.block),
+    ),
+
+    else: $ => seq(
+      'else',
+      $.block,
+    ),
+
     switch: $ => seq(
       'switch',
       $._block_start,
@@ -581,19 +592,19 @@ module.exports = grammar({
 
     try: $ => prec.right(PREC.try, seq(
       'try',
-      field('try', $.block),
-      $._block_continue,
-      'catch',
-      field('catch_arg', $.identifier),
-      field('catch', $.block),
-      optional(
-        seq(
-          $._block_continue,
-          'finally',
-          field('finally', $.block),
-        )
-      )
+      $.block,
     )),
+
+    catch: $ => seq(
+      'catch',
+      $.identifier,
+      $.block,
+    ),
+
+    finally: $ => seq(
+      'finally',
+      $.block,
+    )
   }
 });
 

diff --git a/src/scanner.c b/src/scanner.c
@@ -44,6 +44,8 @@ enum TokenType {
 
 #define VEC_POP(vec) (vec).len--;
 
+#define VEC_SIZE(vec) (vec).len
+
 #define VEC_BACK(vec) ((vec).data[(vec).len - 1])
 
 #define VEC_FREE(vec)                                                                    \
@@ -92,9 +94,7 @@ typedef struct {
 } Scanner;
 
 static void initialize_scanner(Scanner* scanner) {
-  // Ensure that the scanner is initialized with an indent at 0
   VEC_CLEAR(scanner->indents);
-  VEC_PUSH(scanner->indents, 0);
   VEC_CLEAR(scanner->quotes);
   scanner->block_level_just_changed = false;
   scanner->in_string = false;
@@ -273,10 +273,22 @@ bool tree_sitter_koto_external_scanner_scan(
       skip_whitespace(lexer);
     }
 
-    const uint16_t block_indent = VEC_BACK(scanner->indents);
     const uint16_t column = lexer->get_column(lexer);
-    const bool block_just_ended = scanner->block_level_just_changed;
+
+    // Initial block detection
+    if (valid_symbols[BLOCK_START] && VEC_SIZE(scanner->indents) == 0) {
+      printf(">>>> initial block start: %u\n", column);
+      VEC_PUSH(scanner->indents, column);
+      scanner->block_level_just_changed = true;
+      lexer->result_symbol = BLOCK_START;
+      return true;
+    }
+
+    const uint16_t block_indent
+        = VEC_SIZE(scanner->indents) > 0 ? VEC_BACK(scanner->indents) : 0;
+    const bool block_just_changed = scanner->block_level_just_changed;
     scanner->block_level_just_changed = false;
+    const bool eof = lexer->eof(lexer);
 
     printf(
         "scanner.scan: column: %u, block_indent: %u num_indents: %u newline: %i\n",
@@ -315,16 +327,16 @@ bool tree_sitter_koto_external_scanner_scan(
     }
     // Block continue?
     else if (
-        valid_symbols[BLOCK_CONTINUE] && (newline || block_just_ended)
+        valid_symbols[BLOCK_CONTINUE] && !eof && (newline || block_just_changed)
         && column == block_indent) {
       printf(">>>> block continue: %u\n", column);
       lexer->result_symbol = BLOCK_CONTINUE;
       return true;
     }
     // Block end?
     else if (
-        valid_symbols[BLOCK_END] && (newline || block_just_ended)
-        && column < block_indent) {
+        valid_symbols[BLOCK_END]
+        && (eof || (newline || block_just_changed) && column < block_indent)) {
       printf(">>>> block end: %u\n", column);
       VEC_POP(scanner->indents);
       scanner->block_level_just_changed = true;

diff --git a/test/corpus/conditionals.txt b/test/corpus/conditionals.txt
@@ -37,13 +37,15 @@ else
       (number)
     )
     then: (block (identifier) (identifier))
-    else_if_condition: (comparison_op
+  )
+  (else_if
+    condition: (comparison_op
       (binary_op (number) (number))
       (number)
     )
-    else_if_then: (block (identifier))
-    else: (block (string))
+    then: (block (identifier))
   )
+  (else (block (string)))
 )
 
 ===========================================================================================
@@ -64,29 +66,50 @@ else
 
 (module
   (if
-    condition:
-      (comparison_op
-        (binary_op (number) (number))
-        (number)
-      )
+    condition: (comparison_op
+      (binary_op (number) (number))
+      (number)
+    )
     then: (block
       (if
         condition: (comparison_op
           (binary_op (number) (number))
           (number)
         )
         then: (block (identifier))
-        else_if_condition: (comparison_op
+      )
+      (else_if
+        condition: (comparison_op
           (binary_op (number) (number))
           (number)
         )
-        else_if_then: (block (identifier))
+        then: (block (identifier))
       )
     )
-    else_if_condition: (true)
-    else_if_then: (block (string))
-    else: (block (identifier))
   )
+  (else_if
+    condition: (true)
+    then: (block (string))
+  )
+  (else (block (identifier)))
+)
+
+===========================================================================================
+if_block_followed_by_expression
+===========================================================================================
+
+if foo
+  bar
+99
+
+---
+
+(module
+  (if
+    condition: (identifier)
+    then: (block (identifier))
+  )
+  (number)
 )
 
 ===========================================================================================
@@ -169,6 +192,33 @@ switch
   )
 )
 
+===========================================================================================
+switch_followed_by_expression
+===========================================================================================
+
+switch
+  foo then
+    bar
+  else
+    99
+123
+
+---
+
+(module
+  (switch
+    (switch_arm
+      condition: (identifier)
+      then: (block
+        (identifier)
+      )
+    )
+    else: (block (number))
+  )
+  (number)
+)
+
+
 ===========================================================================================
 match_single_line_arms
 ===========================================================================================

diff --git a/test/corpus/expressions.txt b/test/corpus/expressions.txt
@@ -286,17 +286,19 @@ finally
 
 (module
   (try
-    try: (block
+    (block
       (throw (chain start: (identifier) call: (tuple)))
     )
-    catch_arg: (identifier)
-    catch: (block
+  )
+  (catch
+    (identifier)
+    (block
       (throw (identifier))
     )
-    finally: (block 
-      (assign (identifier) (number))
-    )
   )
+  (finally (block
+    (assign (identifier) (number))
+  ))
 )
 
 ===========================================================================================
@@ -318,13 +320,53 @@ x = ||
     (identifier)
     (function
       body: (block
-        (try
-          try: (block (number))
-          catch_arg: (identifier)
-          catch: (block (number))
-          finally: (block (number))
+        (try (block (number)))
+        (catch
+          (identifier)
+          (block (number))
         )
+        (finally (block (number)))
       )
     )
   )
 )
+
+===========================================================================================
+try_catch_no_finally
+===========================================================================================
+
+try
+  foo()
+catch e
+  bar()
+
+---
+
+(module
+  (try (block (chain start: (identifier) call: (tuple))))
+  (catch 
+    (identifier)
+    (block
+      (chain start: (identifier) call: (tuple))
+    )
+  )
+)
+
+===========================================================================================
+try_catch_followed_by_expression
+===========================================================================================
+
+try
+  123
+catch error
+  456
+'hello'
+
+---
+
+(module
+  (try (block (number)))
+  (catch (identifier) (block (number)))
+  (string)
+)
+