diff --git a/albero/languages/java.py b/albero/languages/java.py index a332197..f6e62c6 100644 --- a/albero/languages/java.py +++ b/albero/languages/java.py @@ -88,6 +88,7 @@ "character_literal": "String", "escape_sequence": "String", "string_fragment": "String", + "ERROR": "Error", } diff --git a/albero/languages/python.py b/albero/languages/python.py index ddb7855..27f9bef 100644 --- a/albero/languages/python.py +++ b/albero/languages/python.py @@ -76,6 +76,7 @@ "string_content": "String", "string_end": "String", "string_start": "String", + "ERROR": "Error", } diff --git a/albero/tree_sitter_funcs.py b/albero/tree_sitter_funcs.py index a693c7a..f7c5dc1 100644 --- a/albero/tree_sitter_funcs.py +++ b/albero/tree_sitter_funcs.py @@ -35,7 +35,7 @@ def node_to_tokens( visited_nodes.add(node.id) - if node.child_count != 0: + if node.child_count != 0 and node.type != "ERROR": continue if node.type not in mapping: diff --git a/tests/test-languages/python_test_file.py b/tests/test-languages/python_test_file.py index cedc7e1..c7908b3 100644 --- a/tests/test-languages/python_test_file.py +++ b/tests/test-languages/python_test_file.py @@ -87,3 +87,5 @@ def break_func(self): pass finally: # finally pass + +def unfinished diff --git a/tests/test-languages/test_python.py b/tests/test-languages/test_python.py index 8a79655..7772315 100644 --- a/tests/test-languages/test_python.py +++ b/tests/test-languages/test_python.py @@ -287,4 +287,7 @@ def test_basic_usage(): ((88, 7), 1, "Punctuation"), ((88, 10), 9, "Comment"), ((89, 4), 4, "Keyword"), + ((91, 0), 3, "Keyword"), + ((91, 0), 14, "Error"), + ((91, 4), 10, "Identifier"), ]