Use token_tools instead of tokens file (#7)

* Use token_tools instead of tokens file

* Formatting

albero/__init__.py

@@ -2,7 +2,8 @@
 
 beartype_this_package()
 
+from token_tools import GENERIC_TOKENS, Token  # noqa: F401, E402
+
 from .languages import get_lang, get_mapping  # noqa: F401, E402
 from .misc import AlberoException, lang_from_so  # noqa: F401, E402
-from .tokens import Token, generic_tokens  # noqa: F401, E402
 from .tree_sitter import TreeSitterHighlighter  # noqa: F401, E402

albero/tree_sitter.py

@@ -1,9 +1,9 @@
 from logging import Logger, getLogger
 
+from token_tools import Token, only_tokens_in_text_range
 from tree_sitter import Language, Parser, Tree
 
 from .misc import AlberoException, normal_text_range
-from .tokens import Token, only_tokens_in_text_range
 from .tree_sitter_funcs import edit_tree, node_to_tokens
 
 TreeAndParser = tuple[Tree, Parser, str]  # Tree, Parser, code

albero/tree_sitter_funcs.py

@@ -1,9 +1,8 @@
 from logging import Logger
 
+from token_tools import Token, merge_tokens
 from tree_sitter import Node, Parser, Tree, TreeCursor
 
-from .tokens import Token, merge_tokens
-
 
 def node_to_tokens(
     root_node: Node | Tree, mapping: dict[str, str], logger: Logger

docs/source/variables.rst

@@ -2,18 +2,9 @@
 Variables
 =========
 
-.. _Token Overview:
+.. _Token and GENERIC_TOKENS Overview:
 
-``Token``
-*********
+``Token`` and ``GENERIC_TOKENS``
+********************************
 
-The ``Token`` type is, in reality, just a type alias of ``tuple[tuple[int, int], int, str]``. Despite simply being a ``tuple``, the ``Token`` is the most frequently returned data type. That being said, what data does it signify?
-
-The ``Token`` type contains three parts: the start index, its length, and its type. The start index is that ``tuple`` at the beginning of the main ``tuple`` and the first index of that is the line the ``Token`` takes place on and the second is the column. ``Token``'s start at index ``1, 0`` so you may need to do a -1 or a +1 depending on how you need to use this data. The second ``int`` is the length of the ``Token`  and the ``str`` is the type. You will use these very often so its never a bad idea to get  familiar with them.
-
-.. _Generic Tokens Overview:
-
-``generic_tokens``
-******************
-
-The ``GENERIC_TOKENS`` ``list`` provides all the generic ``Token`` types you will encounter when using ``Albero``. Simply print this out and you will know all possible ``Token`` types and will never be surprised by a random ``Token`` type again.
+See `Token Tools Documentation <https://token-tools.readthedocs.io/en/stable/variables/#variables>`_ for details. Note that ``Token`` is a very common return type used by ``Salve``.

examples/example_mapping_creation.py

@@ -2,7 +2,7 @@
 
 from tree_sitter import Language
 
-from albero import Token, TreeSitterHighlighter, generic_tokens, get_lang
+from albero import GENERIC_TOKENS, Token, TreeSitterHighlighter, get_lang
 
 # Logging
 basicConfig(
@@ -12,7 +12,7 @@
 
 # Useful stuff for highlighting
 archaic_lang: Language = get_lang("archaic_language")
-print(generic_tokens)
+print(GENERIC_TOKENS)
 custom_mapping: dict[str, str] = {
     "xyz_tree_sitter_stuff": "token_in_generic_tokens"  # See python mapping in albero source
 }

requirements-dev.txt

@@ -1,6 +1,7 @@
 # Normal requirements
 beartype
 tree-sitter
+token_tools
 
 # Languages
 tree-sitter-css

requirements.txt

@@ -1,5 +1,6 @@
 beartype
 tree-sitter
+token_tools
 
 # Languages
 tree-sitter-css