From bc96805dc8fb813ac4b2fe4e65da79897bf275c4 Mon Sep 17 00:00:00 2001 From: Daniel Biehl Date: Mon, 25 Dec 2023 16:00:50 +0100 Subject: [PATCH] refactor: move most of langserver...ast_utils to robotcode.robot.utils.ast --- CHANGELOG.md | 18 + .../robotframework/diagnostics/library_doc.py | 11 +- .../parts/code_action_documentation.py | 31 +- .../parts/code_action_helper_mixin.py | 11 +- .../parts/code_action_quick_fixes.py | 83 ++-- .../parts/code_action_refactor.py | 28 +- .../robotframework/parts/codelens.py | 2 +- .../robotframework/parts/completion.py | 4 +- .../robotframework/parts/debugging_utils.py | 12 +- .../robotframework/parts/diagnostics.py | 10 +- .../robotframework/parts/document_symbols.py | 3 +- .../robotframework/parts/documents_cache.py | 2 +- .../robotframework/parts/goto.py | 4 +- .../robotframework/parts/hover.py | 8 +- .../robotframework/parts/inlay_hint.py | 6 +- .../robotframework/parts/inline_value.py | 15 +- .../robotframework/parts/references.py | 15 +- .../robotframework/parts/rename.py | 33 +- .../robotframework/parts/selection_range.py | 13 +- .../robotframework/parts/signature_help.py | 18 +- .../robotframework/utils/ast_utils.py | 387 +----------------- .../robot/src/robotcode/robot/utils/ast.py | 23 +- 22 files changed, 137 insertions(+), 600 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f241306bc..e5cdaf1da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,24 @@ All notable changes to this project will be documented in this file. See [conventional commits](https://www.conventionalcommits.org/) for commit guidelines. +## [unreleased] + +### Refactor + +- **langserver:** Remove async from robotcode langserver analytics ([1ff1e44](https://github.com/d-biehl/robotcode/commit/1ff1e44d91f64671e108b35f8fe9f7005a72be5d)) +- **langserver:** Remove AsyncVisitor code ([c9aaa60](https://github.com/d-biehl/robotcode/commit/c9aaa6088a86c67b8dec5cb9794efe6c339cc6a5)) +- **robotcode:** Move threaded decorator to the new core.utils.threading module ([96b897b](https://github.com/d-biehl/robotcode/commit/96b897b63bdc13aaa0e383a12e74399e0f8caa86)) +- Remove some unneeded code ([65e1775](https://github.com/d-biehl/robotcode/commit/65e1775fd380b7e6b67a88c327f8961929e9cafb)) + + +### Testing + +- Add RFW 7.0 to devel and test matrix ([cd35020](https://github.com/d-biehl/robotcode/commit/cd3502086b1505b53a89047d27ca097d3dfce07b)) +- Try to stablize regression tests ([19419b3](https://github.com/d-biehl/robotcode/commit/19419b3c10f0a7e079d6b6b7466516f4ec756f88)) +- Enable RFW 7.0 tests in github workflow ([6a39f66](https://github.com/d-biehl/robotcode/commit/6a39f66f6113ef8e1437a3bf2ae10d3de6fe0203)) +- Another try to stabilize regression tests ([91d4d48](https://github.com/d-biehl/robotcode/commit/91d4d482fdcecc65270f98f715760576ea9f7544)) + + ## [0.68.1](https://github.com/d-biehl/robotcode/compare/v0.68.0..v0.68.1) - 2023-12-21 ### Bug Fixes diff --git a/packages/language_server/src/robotcode/language_server/robotframework/diagnostics/library_doc.py b/packages/language_server/src/robotcode/language_server/robotframework/diagnostics/library_doc.py index 367b25b3e..9c78c9b73 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/diagnostics/library_doc.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/diagnostics/library_doc.py @@ -33,18 +33,13 @@ cast, ) +from robot.parsing.lexer.tokens import Token from robotcode.core.lsp.types import Position, Range from robotcode.robot.utils import get_robot_version +from robotcode.robot.utils.ast import get_variable_token, range_from_token, strip_variable_token from robotcode.robot.utils.markdownformatter import MarkDownFormatter -from ..utils.ast_utils import ( - HasError, - HasErrors, - Token, - get_variable_token, - range_from_token, - strip_variable_token, -) +from ..utils.ast_utils import HasError, HasErrors from ..utils.match import normalize, normalize_namespace from .entities import ( ArgumentDefinition, diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/code_action_documentation.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/code_action_documentation.py index e2e338511..d0e408cf1 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/code_action_documentation.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/code_action_documentation.py @@ -17,40 +17,23 @@ from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union, cast from urllib.parse import parse_qs, urlparse -from robotcode.core.lsp.types import ( - CodeAction, - CodeActionContext, - CodeActionKind, - Command, - Range, -) +from robot.parsing.lexer.tokens import Token +from robotcode.core.lsp.types import CodeAction, CodeActionContext, CodeActionKind, Command, Range from robotcode.core.uri import Uri from robotcode.core.utils.dataclasses import CamelSnakeMixin from robotcode.core.utils.logging import LoggingDescriptor from robotcode.core.utils.net import find_free_port from robotcode.core.utils.threading import threaded from robotcode.jsonrpc2.protocol import rpc_method +from robotcode.robot.utils.ast import get_node_at_position, range_from_token from ...common.decorators import code_action_kinds, language_id from ...common.text_document import TextDocument -from ..configuration import ( - DocumentationServerConfig, -) +from ..configuration import DocumentationServerConfig from ..diagnostics.entities import LibraryEntry -from ..diagnostics.library_doc import ( - get_library_doc, - get_robot_library_html_doc_str, - resolve_robot_variables, -) +from ..diagnostics.library_doc import get_library_doc, get_robot_library_html_doc_str, resolve_robot_variables from ..diagnostics.model_helper import ModelHelperMixin -from ..diagnostics.namespace import ( - Namespace, -) -from ..utils.ast_utils import ( - Token, - get_node_at_position, - range_from_token, -) +from ..diagnostics.namespace import Namespace from .protocol_part import RobotLanguageServerProtocolPart if TYPE_CHECKING: @@ -277,7 +260,7 @@ async def collect( namespace = self.parent.documents_cache.get_namespace(document) model = self.parent.documents_cache.get_model(document, False) - node = await get_node_at_position(model, range.start) + node = get_node_at_position(model, range.start) if context.only and isinstance(node, (LibraryImport, ResourceImport)): if CodeActionKind.SOURCE.value in context.only and range in range_from_token( diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/code_action_helper_mixin.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/code_action_helper_mixin.py index 49c75062f..b1369cbde 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/code_action_helper_mixin.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/code_action_helper_mixin.py @@ -9,12 +9,11 @@ Position, Range, ) -from robotcode.language_server.common.text_document import TextDocument -from robotcode.language_server.robotframework.diagnostics.namespace import Namespace -from robotcode.language_server.robotframework.utils.ast_utils import ( - range_from_node, -) -from robotcode.language_server.robotframework.utils.async_ast import Visitor +from robotcode.robot.utils.ast import range_from_node + +from ...common.text_document import TextDocument +from ..diagnostics.namespace import Namespace +from ..utils.async_ast import Visitor SHOW_DOCUMENT_SELECT_AND_RENAME_COMMAND = "_robotcode.codeActionShowDocumentSelectAndRename" diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/code_action_quick_fixes.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/code_action_quick_fixes.py index 7a3f00bd3..0db17125c 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/code_action_quick_fixes.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/code_action_quick_fixes.py @@ -2,9 +2,25 @@ from collections import defaultdict from dataclasses import dataclass -from string import Template +from string import Template as StringTemplate from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Tuple, Union, cast +from robot.parsing.lexer.tokens import Token +from robot.parsing.model.blocks import Keyword, TestCase, VariableSection +from robot.parsing.model.statements import ( + Arguments, + Documentation, + Fixture, + KeywordCall, + KeywordName, + Statement, + Template, + TestCaseName, + TestTemplate, + Variable, +) +from robot.utils.escaping import split_from_equals +from robot.variables.search import contains_variable from robotcode.core.lsp.types import ( AnnotatedTextEdit, ChangeAnnotation, @@ -23,14 +39,8 @@ from robotcode.core.utils.dataclasses import as_dict, from_dict from robotcode.core.utils.inspect import iter_methods from robotcode.core.utils.logging import LoggingDescriptor - -from ...common.decorators import code_action_kinds, language_id -from ...common.text_document import TextDocument -from ..diagnostics.errors import DIAGNOSTICS_SOURCE_NAME, Error -from ..diagnostics.model_helper import ModelHelperMixin -from ..utils.ast_utils import ( +from robotcode.robot.utils.ast import ( FirstAndLastRealStatementFinder, - Token, get_node_at_position, get_nodes_at_position, get_tokens_at_position, @@ -38,6 +48,11 @@ range_from_node, range_from_token, ) + +from ...common.decorators import code_action_kinds, language_id +from ...common.text_document import TextDocument +from ..diagnostics.errors import DIAGNOSTICS_SOURCE_NAME, Error +from ..diagnostics.model_helper import ModelHelperMixin from .code_action_helper_mixin import ( SHOW_DOCUMENT_SELECT_AND_RENAME_COMMAND, CodeActionDataBase, @@ -55,7 +70,7 @@ class CodeActionData(CodeActionDataBase): diagnostics_code: Optional[Union[int, str]] = None -KEYWORD_WITH_ARGS_TEMPLATE = Template( +KEYWORD_WITH_ARGS_TEMPLATE = StringTemplate( """\ ${name} [Arguments] ${args} @@ -64,7 +79,7 @@ class CodeActionData(CodeActionDataBase): """ ) -KEYWORD_TEMPLATE = Template( +KEYWORD_TEMPLATE = StringTemplate( """\ ${name} # TODO: implement keyword "${name}". @@ -113,13 +128,6 @@ async def resolve(self, sender: Any, code_action: CodeAction) -> Optional[CodeAc async def code_action_create_keyword( self, document: TextDocument, range: Range, context: CodeActionContext ) -> Optional[List[Union[Command, CodeAction]]]: - from robot.parsing.model.statements import ( - Fixture, - KeywordCall, - Template, - TestTemplate, - ) - result: List[Union[Command, CodeAction]] = [] if (context.only and CodeActionKind.QUICK_FIX in context.only) or context.trigger_kind in [ @@ -135,7 +143,7 @@ async def code_action_create_keyword( if d.source == DIAGNOSTICS_SOURCE_NAME and d.code == Error.KEYWORD_NOT_FOUND ): disabled = None - node = await get_node_at_position(model, diagnostic.range.start) + node = get_node_at_position(model, diagnostic.range.start) if isinstance(node, (KeywordCall, Fixture, TestTemplate, Template)): tokens = get_tokens_at_position(node, diagnostic.range.start) @@ -179,22 +187,12 @@ async def code_action_create_keyword( async def resolve_code_action_create_keyword( self, code_action: CodeAction, data: CodeActionData ) -> Optional[CodeAction]: - from robot.parsing.lexer import Token as RobotToken - from robot.parsing.model.statements import ( - Fixture, - KeywordCall, - Template, - TestTemplate, - ) - from robot.utils.escaping import split_from_equals - from robot.variables.search import contains_variable - document = self.parent.documents.get(data.document_uri) if document is None: return None model = self.parent.documents_cache.get_model(document, False) - node = await get_node_at_position(model, data.range.start) + node = get_node_at_position(model, data.range.start) if isinstance(node, (KeywordCall, Fixture, TestTemplate, Template)): tokens = get_tokens_at_position(node, data.range.start) @@ -224,7 +222,7 @@ async def resolve_code_action_create_keyword( arguments = [] - for t in node.get_tokens(RobotToken.ARGUMENT): + for t in node.get_tokens(Token.ARGUMENT): name, value = split_from_equals(cast(Token, t).value) if value is not None and not contains_variable(name, "$@&%"): arguments.append(f"${{{name}}}") @@ -355,9 +353,6 @@ async def resolve_code_action_disable_robotcode_diagnostics_for_line( async def code_action_create_local_variable( self, document: TextDocument, range: Range, context: CodeActionContext ) -> Optional[List[Union[Command, CodeAction]]]: - from robot.parsing.model.blocks import Keyword, TestCase - from robot.parsing.model.statements import Documentation, Fixture, Statement, Template, TestCaseName - result: List[Union[Command, CodeAction]] = [] if (context.only and CodeActionKind.QUICK_FIX in context.only) or context.trigger_kind in [ @@ -374,7 +369,7 @@ async def code_action_create_local_variable( and diagnostic.range.start.character < diagnostic.range.end.character ): model = self.parent.documents_cache.get_model(document, False) - nodes = await get_nodes_at_position(model, range.start) + nodes = get_nodes_at_position(model, range.start) if not any(n for n in nodes if isinstance(n, (Keyword, TestCase))): continue @@ -413,16 +408,13 @@ async def code_action_create_local_variable( async def resolve_code_action_create_local_variable( self, code_action: CodeAction, data: CodeActionData ) -> Optional[CodeAction]: - from robot.parsing.model.blocks import Keyword, TestCase - from robot.parsing.model.statements import Documentation, Fixture, Statement, Template - if data.range.start.line == data.range.end.line and data.range.start.character <= data.range.end.character: document = self.parent.documents.get(data.document_uri) if document is None: return None model = self.parent.documents_cache.get_model(document, False) - nodes = await get_nodes_at_position(model, data.range.start) + nodes = get_nodes_at_position(model, data.range.start) if not any(n for n in nodes if isinstance(n, (Keyword, TestCase))): return None @@ -510,16 +502,13 @@ async def code_action_create_suite_variable( async def resolve_code_action_create_suite_variable( self, code_action: CodeAction, data: CodeActionData ) -> Optional[CodeAction]: - from robot.parsing.model.blocks import VariableSection - from robot.parsing.model.statements import Variable - if data.range.start.line == data.range.end.line and data.range.start.character <= data.range.end.character: document = self.parent.documents.get(data.document_uri) if document is None: return None model = self.parent.documents_cache.get_model(document, False) - nodes = await get_nodes_at_position(model, data.range.start) + nodes = get_nodes_at_position(model, data.range.start) node = nodes[-1] if nodes else None @@ -603,8 +592,6 @@ async def resolve_code_action_create_suite_variable( async def code_action_add_argument( self, document: TextDocument, range: Range, context: CodeActionContext ) -> Optional[List[Union[Command, CodeAction]]]: - from robot.parsing.model.blocks import Keyword - result: List[Union[Command, CodeAction]] = [] if (context.only and CodeActionKind.QUICK_FIX in context.only) or context.trigger_kind in [ @@ -628,7 +615,7 @@ async def code_action_add_argument( continue model = self.parent.documents_cache.get_model(document, False) - nodes = await get_nodes_at_position(model, range.start) + nodes = get_nodes_at_position(model, range.start) if not any(n for n in nodes if isinstance(n, Keyword)): continue @@ -654,10 +641,6 @@ async def code_action_add_argument( async def resolve_code_action_add_argument( self, code_action: CodeAction, data: CodeActionData ) -> Optional[CodeAction]: - from robot.parsing.lexer.tokens import Token - from robot.parsing.model.blocks import Keyword - from robot.parsing.model.statements import Arguments, Documentation, KeywordName, Statement - if data.range.start.line == data.range.end.line and data.range.start.character <= data.range.end.character: document = self.parent.documents.get(data.document_uri) if document is None: @@ -668,7 +651,7 @@ async def resolve_code_action_add_argument( return None model = self.parent.documents_cache.get_model(document, False) - nodes = await get_nodes_at_position(model, data.range.start) + nodes = get_nodes_at_position(model, data.range.start) keyword = next((n for n in nodes if isinstance(n, Keyword)), None) if keyword is None: diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/code_action_refactor.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/code_action_refactor.py index 6212b8830..a1d1924f7 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/code_action_refactor.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/code_action_refactor.py @@ -27,18 +27,18 @@ from robotcode.core.utils.inspect import iter_methods from robotcode.core.utils.logging import LoggingDescriptor from robotcode.robot.utils import get_robot_version - -from ...common.decorators import code_action_kinds, language_id -from ...common.text_document import TextDocument -from ..diagnostics.model_helper import ModelHelperMixin -from ..utils import ast_utils -from ..utils.ast_utils import ( - BodyBlock, +from robotcode.robot.utils.ast import ( get_node_at_position, get_nodes_at_position, + iter_nodes, range_from_node, range_from_token, ) + +from ...common.decorators import code_action_kinds, language_id +from ...common.text_document import TextDocument +from ..diagnostics.model_helper import ModelHelperMixin +from ..utils.ast_utils import BodyBlock from .code_action_helper_mixin import SHOW_DOCUMENT_SELECT_AND_RENAME_COMMAND, CodeActionDataBase, CodeActionHelperMixin from .protocol_part import RobotLanguageServerProtocolPart @@ -169,7 +169,7 @@ def get_valid_nodes_in_range(self, model: ast.AST, range: Range, also_return: bo result = [] blocks: List[BodyBlock] = [] - for node in ast_utils.iter_nodes(model): + for node in iter_nodes(model): if isinstance(node, Block) and isinstance(node, BodyBlock): blocks.append(node) @@ -213,7 +213,7 @@ def get_valid_nodes_in_range(self, model: ast.AST, range: Range, also_return: bo results = [] for block in [model, *blocks]: - sub = [n for n in result if n in ast_utils.iter_nodes(block, False)] + sub = [n for n in result if n in iter_nodes(block, False)] if sub: results.append(sub) @@ -266,7 +266,7 @@ async def code_action_surround( return None model = self.parent.documents_cache.get_model(document, False) - start_nodes = await get_nodes_at_position(model, range.start) + start_nodes = get_nodes_at_position(model, range.start) enabled = False insert_range = None @@ -412,7 +412,7 @@ async def code_action_assign_result_to_variable( ] ): model = self.parent.documents_cache.get_model(document, False) - node = await get_node_at_position(model, range.start) + node = get_node_at_position(model, range.start) if not isinstance(node, KeywordCall) or node.assign: return None @@ -458,7 +458,7 @@ async def resolve_code_action_assign_result_to_variable( return None model = self.parent.documents_cache.get_model(document, False) - nodes = await get_nodes_at_position(model, range.start) + nodes = get_nodes_at_position(model, range.start) if not nodes: return None node = nodes[-1] @@ -516,7 +516,7 @@ async def code_action_extract_keyword( return None model = self.parent.documents_cache.get_model(document, False) - start_nodes = await get_nodes_at_position(model, range.start) + start_nodes = get_nodes_at_position(model, range.start) enabled = False insert_range = None @@ -576,7 +576,7 @@ async def resolve_code_action_extract_keyword( if kw_counter > 100: return None - start_nodes = await get_nodes_at_position(model, data.range.start) + start_nodes = get_nodes_at_position(model, data.range.start) block = next((n for n in start_nodes if isinstance(n, (Keyword, TestCase))), None) if block is None: return None diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/codelens.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/codelens.py index 340fa8a2c..b1f77b620 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/codelens.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/codelens.py @@ -6,13 +6,13 @@ from robotcode.core.async_tools import create_sub_task from robotcode.core.lsp.types import CodeLens, Command from robotcode.core.utils.logging import LoggingDescriptor +from robotcode.robot.utils.ast import range_from_token from ...common.decorators import language_id from ...common.text_document import TextDocument from ..configuration import AnalysisConfig from ..diagnostics.library_doc import KeywordDoc from ..diagnostics.model_helper import ModelHelperMixin -from ..utils.ast_utils import range_from_token from ..utils.async_ast import Visitor from .protocol_part import RobotLanguageServerProtocolPart diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/completion.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/completion.py index 9a2162dd3..27dcc3ed9 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/completion.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/completion.py @@ -316,7 +316,7 @@ async def collect( ) -> Union[List[CompletionItem], CompletionList, None]: start = time.monotonic() try: - result_nodes = await get_nodes_at_position(self.model, position, include_end=True) + result_nodes = get_nodes_at_position(self.model, position, include_end=True) result_nodes.reverse() @@ -1065,7 +1065,7 @@ async def complete_default( elif position.character == 0: if not nodes_at_position and position.line > 0: - nodes_at_line_before = await get_nodes_at_position(self.model, Position(position.line - 1, 0)) + nodes_at_line_before = get_nodes_at_position(self.model, Position(position.line - 1, 0)) if nodes_at_line_before and any(isinstance(n, SettingSection) for n in nodes_at_line_before): return [ *await self.create_settings_completion_items(None), diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/debugging_utils.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/debugging_utils.py index 02a4ea6f2..6002ae2e3 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/debugging_utils.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/debugging_utils.py @@ -3,19 +3,15 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Optional +from robot.parsing.model.statements import Statement from robotcode.core.lsp.types import Position, Range, TextDocumentIdentifier from robotcode.core.utils.dataclasses import CamelSnakeMixin from robotcode.core.utils.logging import LoggingDescriptor from robotcode.core.utils.threading import threaded from robotcode.jsonrpc2.protocol import rpc_method +from robotcode.robot.utils.ast import get_nodes_at_position, get_tokens_at_position, range_from_token from ..diagnostics.model_helper import ModelHelperMixin -from ..utils.ast_utils import ( - HasTokens, - get_nodes_at_position, - get_tokens_at_position, - range_from_token, -) from .protocol_part import RobotLanguageServerProtocolPart if TYPE_CHECKING: @@ -59,10 +55,10 @@ async def _get_evaluatable_expression( namespace = self.parent.documents_cache.get_namespace(document) model = self.parent.documents_cache.get_model(document, False) - nodes = await get_nodes_at_position(model, position) + nodes = get_nodes_at_position(model, position) node = nodes[-1] - if not isinstance(node, HasTokens): + if not isinstance(node, Statement): return None token = get_tokens_at_position(node, position)[-1] diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/diagnostics.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/diagnostics.py index f247015be..2c2a981dc 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/diagnostics.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/diagnostics.py @@ -4,11 +4,12 @@ import asyncio from typing import TYPE_CHECKING, Any, List, Optional +from robot.parsing.lexer.tokens import Token from robotcode.core.async_tools import create_sub_task from robotcode.core.lsp.types import Diagnostic, DiagnosticSeverity, DiagnosticTag, Position, Range from robotcode.core.uri import Uri from robotcode.core.utils.logging import LoggingDescriptor -from robotcode.language_server.robotframework.utils.ast_utils import HasError, HasErrors, iter_nodes +from robotcode.robot.utils.ast import iter_nodes, range_from_node, range_from_token from ...common.decorators import language_id from ...common.parts.diagnostics import DiagnosticsResult @@ -16,12 +17,7 @@ from ..configuration import AnalysisConfig from ..diagnostics.entities import ArgumentDefinition from ..diagnostics.namespace import Namespace -from ..utils.ast_utils import ( - HeaderAndBodyBlock, - Token, - range_from_node, - range_from_token, -) +from ..utils.ast_utils import HasError, HasErrors, HeaderAndBodyBlock if TYPE_CHECKING: from ..protocol import RobotLanguageServerProtocol diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/document_symbols.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/document_symbols.py index b56814d9c..d51832f7d 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/document_symbols.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/document_symbols.py @@ -4,12 +4,13 @@ import itertools from typing import TYPE_CHECKING, Any, List, Optional, Union, cast +from robot.parsing.lexer.tokens import Token from robotcode.core.lsp.types import DocumentSymbol, SymbolInformation, SymbolKind from robotcode.core.utils.logging import LoggingDescriptor +from robotcode.robot.utils.ast import range_from_node, range_from_token, tokenize_variables from ...common.decorators import language_id from ...common.text_document import TextDocument -from ..utils.ast_utils import Token, range_from_node, range_from_token, tokenize_variables from ..utils.async_ast import Visitor from .protocol_part import RobotLanguageServerProtocolPart diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/documents_cache.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/documents_cache.py index eaffa0840..4279fefea 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/documents_cache.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/documents_cache.py @@ -17,6 +17,7 @@ cast, ) +from robot.parsing.lexer.tokens import Token from robotcode.core.event import tasking_event from robotcode.core.lsp.types import MessageType from robotcode.core.uri import Uri @@ -30,7 +31,6 @@ from ..diagnostics.imports_manager import ImportsManager from ..diagnostics.namespace import DocumentType, Namespace from ..languages import Languages -from ..utils.ast_utils import Token from .protocol_part import RobotLanguageServerProtocolPart if TYPE_CHECKING: diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/goto.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/goto.py index eb692dc78..e81078921 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/goto.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/goto.py @@ -12,12 +12,10 @@ from robotcode.core.lsp.types import Location, LocationLink, Position, Range from robotcode.core.uri import Uri from robotcode.core.utils.logging import LoggingDescriptor +from robotcode.robot.utils.ast import range_from_token from ...common.decorators import language_id from ...common.text_document import TextDocument -from ..utils.ast_utils import ( - range_from_token, -) from .protocol_part import RobotLanguageServerProtocolPart if TYPE_CHECKING: diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/hover.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/hover.py index 1f599dede..8e27ead25 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/hover.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/hover.py @@ -17,16 +17,12 @@ from robotcode.core.lsp.types import Hover, MarkupContent, MarkupKind, Position, Range from robotcode.core.utils.logging import LoggingDescriptor +from robotcode.robot.utils.ast import get_nodes_at_position, range_from_node, range_from_token from robotcode.robot.utils.markdownformatter import MarkDownFormatter from ...common.decorators import language_id from ...common.text_document import TextDocument from ..diagnostics.model_helper import ModelHelperMixin -from ..utils.ast_utils import ( - get_nodes_at_position, - range_from_node, - range_from_token, -) from .protocol_part import RobotLanguageServerProtocolPart if TYPE_CHECKING: @@ -63,7 +59,7 @@ def _find_method(self, cls: Type[Any]) -> Optional[_HoverMethod]: async def collect(self, sender: Any, document: TextDocument, position: Position) -> Optional[Hover]: model = self.parent.documents_cache.get_model(document) - result_nodes = await get_nodes_at_position(model, position) + result_nodes = get_nodes_at_position(model, position) if not result_nodes: return None diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/inlay_hint.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/inlay_hint.py index f39a623f5..849a63e8e 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/inlay_hint.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/inlay_hint.py @@ -4,16 +4,16 @@ import asyncio from typing import TYPE_CHECKING, Any, Awaitable, Callable, List, Optional, Type, cast +from robot.parsing.lexer.tokens import Token from robotcode.core.lsp.types import InlayHint, InlayHintKind, Range from robotcode.core.utils.logging import LoggingDescriptor +from robotcode.robot.utils.ast import iter_nodes, range_from_node, range_from_token from ...common.decorators import language_id from ...common.text_document import TextDocument from ..configuration import InlayHintsConfig from ..diagnostics.library_doc import KeywordArgumentKind, KeywordDoc, LibraryDoc from ..diagnostics.namespace import Namespace -from ..utils.ast_utils import Token, range_from_node, range_from_token -from ..utils.async_ast import iter_nodes if TYPE_CHECKING: from ..protocol import RobotLanguageServerProtocol @@ -68,7 +68,7 @@ async def collect(self, sender: Any, document: TextDocument, range: Range) -> Op result: List[InlayHint] = [] - async for node in iter_nodes(model): + for node in iter_nodes(model): node_range = range_from_node(node) if node_range.end < range.start: continue diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/inline_value.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/inline_value.py index 10e35e5a4..912475596 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/inline_value.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/inline_value.py @@ -3,21 +3,16 @@ import ast from typing import TYPE_CHECKING, Any, Iterator, List, Optional, Tuple +from robot.parsing.lexer.tokens import Token +from robot.parsing.model.statements import Statement from robotcode.core.async_itertools import async_dropwhile, async_takewhile from robotcode.core.lsp.types import InlineValue, InlineValueContext, InlineValueEvaluatableExpression, Range from robotcode.core.utils.logging import LoggingDescriptor +from robotcode.robot.utils.ast import get_nodes_at_position, iter_nodes, range_from_node, range_from_token from ...common.decorators import language_id from ...common.text_document import TextDocument from ..diagnostics.model_helper import ModelHelperMixin -from ..utils.ast_utils import ( - HasTokens, - Token, - get_nodes_at_position, - iter_nodes, - range_from_node, - range_from_token, -) from .protocol_part import RobotLanguageServerProtocolPart if TYPE_CHECKING: @@ -53,12 +48,12 @@ async def collect( real_range = Range(range.start, min(range.end, context.stopped_location.end)) - nodes = await get_nodes_at_position(model, context.stopped_location.start) + nodes = get_nodes_at_position(model, context.stopped_location.start) def get_tokens() -> Iterator[Tuple[Token, ast.AST]]: for n in iter_nodes(model): r = range_from_node(n) - if (r.start in real_range or r.end in real_range) and isinstance(n, HasTokens): + if (r.start in real_range or r.end in real_range) and isinstance(n, Statement): for t in n.tokens: yield t, n if r.start > real_range.end: diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/references.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/references.py index 02c8d90bc..dac10bf89 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/references.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/references.py @@ -12,6 +12,7 @@ cast, ) +from robot.parsing.model.statements import Statement from robotcode.core.async_tools import async_event, create_sub_task from robotcode.core.lsp.types import FileEvent, Location, Position, Range, ReferenceContext, WatchKind from robotcode.core.uri import Uri @@ -19,6 +20,7 @@ from robotcode.core.utils.logging import LoggingDescriptor from robotcode.core.utils.threading import threaded from robotcode.robot.utils import get_robot_version +from robotcode.robot.utils.ast import get_nodes_at_position, get_tokens_at_position, iter_nodes, range_from_token from ...common.decorators import language_id from ...common.text_document import TextDocument @@ -35,13 +37,6 @@ LibraryDoc, ) from ..diagnostics.model_helper import ModelHelperMixin -from ..utils.ast_utils import ( - HasTokens, - get_nodes_at_position, - get_tokens_at_position, - iter_nodes, - range_from_token, -) from ..utils.match import normalize from .protocol_part import RobotLanguageServerProtocolPart @@ -109,7 +104,7 @@ def _find_method(self, cls: Type[Any]) -> Optional[_ReferencesMethod]: async def collect( self, sender: Any, document: TextDocument, position: Position, context: ReferenceContext ) -> Optional[List[Location]]: - result_nodes = await get_nodes_at_position(self.parent.documents_cache.get_model(document), position) + result_nodes = get_nodes_at_position(self.parent.documents_cache.get_model(document), position) if not result_nodes: return None @@ -559,11 +554,11 @@ def _references_tags( ) -> Optional[List[Location]]: from robot.parsing.lexer.tokens import Token as RobotToken - tokens = get_tokens_at_position(cast(HasTokens, node), position) + tokens = get_tokens_at_position(cast(Statement, node), position) if not tokens: return None - token = get_tokens_at_position(cast(HasTokens, node), position)[-1] + token = get_tokens_at_position(cast(Statement, node), position)[-1] if token.type in [RobotToken.ARGUMENT] and token.value: return self.find_tag_references(document, token.value) diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/rename.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/rename.py index 1830ceecb..c0badcd36 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/rename.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/rename.py @@ -1,20 +1,10 @@ from __future__ import annotations import ast -from typing import ( - TYPE_CHECKING, - Any, - Awaitable, - Callable, - List, - Optional, - Tuple, - Type, - TypeVar, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Awaitable, Callable, List, Optional, Tuple, Type, TypeVar, Union, cast +from robot.parsing.lexer.tokens import Token +from robot.parsing.model.statements import Statement from robotcode.core.lsp.types import ( AnnotatedTextEdit, ChangeAnnotation, @@ -29,6 +19,7 @@ WorkspaceEdit, ) from robotcode.core.utils.logging import LoggingDescriptor +from robotcode.robot.utils.ast import get_nodes_at_position, get_tokens_at_position, range_from_token from ...common.decorators import language_id from ...common.parts.rename import CantRenameError @@ -36,13 +27,11 @@ from ..diagnostics.entities import VariableDefinition, VariableDefinitionType from ..diagnostics.library_doc import KeywordDoc from ..diagnostics.model_helper import ModelHelperMixin -from ..utils.ast_utils import HasTokens, Token, get_nodes_at_position, get_tokens_at_position, range_from_token from .protocol_part import RobotLanguageServerProtocolPart if TYPE_CHECKING: - from robotcode.language_server.robotframework.protocol import ( - RobotLanguageServerProtocol, - ) + from robotcode.language_server.robotframework.protocol import RobotLanguageServerProtocol + _RenameMethod = Callable[[ast.AST, TextDocument, Position, str], Awaitable[Optional[WorkspaceEdit]]] _PrepareRenameMethod = Callable[[ast.AST, TextDocument, Position], Awaitable[Optional[PrepareRenameResult]]] @@ -82,7 +71,7 @@ async def collect( position: Position, new_name: str, ) -> Optional[WorkspaceEdit]: - result_nodes = await get_nodes_at_position( + result_nodes = get_nodes_at_position( self.parent.documents_cache.get_model(document), position, include_end=True ) @@ -111,7 +100,7 @@ async def collect_prepare( document: TextDocument, position: Position, ) -> Optional[PrepareRenameResult]: - result_nodes = await get_nodes_at_position( + result_nodes = get_nodes_at_position( self.parent.documents_cache.get_model(document), position, include_end=True ) @@ -214,7 +203,7 @@ async def _find_default( node = nodes[-1] - if not isinstance(node, HasTokens): + if not isinstance(node, Statement): return None tokens = get_tokens_at_position(node, position) @@ -567,7 +556,7 @@ async def _prepare_rename_tags( ) -> Optional[PrepareRenameResult]: from robot.parsing.lexer.tokens import Token as RobotToken - tokens = get_tokens_at_position(cast(HasTokens, node), position) + tokens = get_tokens_at_position(cast(Statement, node), position) if not tokens: return None @@ -598,7 +587,7 @@ async def _rename_tags( ) -> Optional[WorkspaceEdit]: from robot.parsing.lexer.tokens import Token as RobotToken - tokens = get_tokens_at_position(cast(HasTokens, node), position) + tokens = get_tokens_at_position(cast(Statement, node), position) if not tokens: return None diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/selection_range.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/selection_range.py index 737925850..b8f660768 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/selection_range.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/selection_range.py @@ -2,19 +2,14 @@ from typing import TYPE_CHECKING, Any, List, Optional +from robot.parsing.model.statements import Statement from robotcode.core.lsp.types import Position, SelectionRange from robotcode.core.utils.logging import LoggingDescriptor +from robotcode.robot.utils.ast import get_nodes_at_position, get_tokens_at_position, range_from_node, range_from_token from ...common.decorators import language_id from ...common.text_document import TextDocument from ..diagnostics.model_helper import ModelHelperMixin -from ..utils.ast_utils import ( - HasTokens, - get_nodes_at_position, - get_tokens_at_position, - range_from_node, - range_from_token, -) from .protocol_part import RobotLanguageServerProtocolPart if TYPE_CHECKING: @@ -38,7 +33,7 @@ async def collect( results: List[SelectionRange] = [] for position in positions: - nodes = await get_nodes_at_position(self.parent.documents_cache.get_model(document, True), position) + nodes = get_nodes_at_position(self.parent.documents_cache.get_model(document, True), position) if not nodes: break @@ -49,7 +44,7 @@ async def collect( if current_range is not None: node = nodes[-1] - if node is not None and isinstance(node, HasTokens): + if node is not None and isinstance(node, Statement): tokens = get_tokens_at_position(node, position, True) if tokens: token = tokens[-1] diff --git a/packages/language_server/src/robotcode/language_server/robotframework/parts/signature_help.py b/packages/language_server/src/robotcode/language_server/robotframework/parts/signature_help.py index 0400bf978..2f395cf0d 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/parts/signature_help.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/parts/signature_help.py @@ -2,18 +2,10 @@ import ast import asyncio -from typing import ( - TYPE_CHECKING, - Any, - Awaitable, - Callable, - Optional, - Sequence, - Tuple, - Type, - cast, -) +from typing import TYPE_CHECKING, Any, Awaitable, Callable, Optional, Sequence, Tuple, Type, cast +from robot.parsing.lexer.tokens import Token +from robot.parsing.model.statements import Statement from robotcode.core.lsp.types import ( MarkupContent, MarkupKind, @@ -24,12 +16,12 @@ SignatureInformation, ) from robotcode.core.utils.logging import LoggingDescriptor +from robotcode.robot.utils.ast import get_node_at_position, get_tokens_at_position, range_from_token from ...common.decorators import language_id, retrigger_characters, trigger_characters from ...common.text_document import TextDocument from ..diagnostics.library_doc import KeywordDoc, LibraryDoc from ..diagnostics.model_helper import ModelHelperMixin -from ..utils.ast_utils import Statement, Token, get_node_at_position, get_tokens_at_position, range_from_token from .protocol_part import RobotLanguageServerProtocolPart if TYPE_CHECKING: @@ -69,7 +61,7 @@ def _find_method(self, cls: Type[Any]) -> Optional[_SignatureHelpMethod]: async def collect( self, sender: Any, document: TextDocument, position: Position, context: Optional[SignatureHelpContext] = None ) -> Optional[SignatureHelp]: - result_node = await get_node_at_position( + result_node = get_node_at_position( self.parent.documents_cache.get_model(document, False), position, include_end=True ) if result_node is None: diff --git a/packages/language_server/src/robotcode/language_server/robotframework/utils/ast_utils.py b/packages/language_server/src/robotcode/language_server/robotframework/utils/ast_utils.py index 8d3099977..41f9d2512 100644 --- a/packages/language_server/src/robotcode/language_server/robotframework/utils/ast_utils.py +++ b/packages/language_server/src/robotcode/language_server/robotframework/utils/ast_utils.py @@ -1,61 +1,6 @@ from __future__ import annotations -import ast -import itertools -from typing import ( - Any, - AsyncIterator, - Iterator, - List, - Optional, - Protocol, - Sequence, - Set, - Tuple, - cast, - runtime_checkable, -) - -from robotcode.core.lsp.types import Position, Range -from robotcode.robot.utils import get_robot_version - -from . import async_ast - - -def iter_nodes(node: ast.AST, descendants: bool = True) -> Iterator[ast.AST]: - for _field, value in ast.iter_fields(node): - if isinstance(value, list): - for item in value: - if isinstance(item, ast.AST): - yield item - if descendants: - yield from iter_nodes(item) - - elif isinstance(value, ast.AST): - yield value - if descendants: - yield from iter_nodes(value) - - -@runtime_checkable -class Token(Protocol): - type: Optional[str] - value: str - lineno: int - col_offset: int - error: Optional[str] - - @property - def end_col_offset(self) -> int: - ... - - def tokenize_variables(self) -> Iterator[Token]: - ... - - -@runtime_checkable -class HasTokens(Protocol): - tokens: Sequence[Token] +from typing import Any, List, Optional, Protocol, runtime_checkable @runtime_checkable @@ -68,37 +13,6 @@ class HasErrors(Protocol): errors: Optional[List[str]] -@runtime_checkable -class Statement(HasTokens, Protocol): - def get_token(self, type: str) -> Optional[Token]: - ... - - def get_tokens(self, *types: str) -> Tuple[Token, ...]: - ... - - def get_value(self, type: str, default: Any = None) -> Any: - ... - - def get_values(self, *types: str) -> Tuple[Any, ...]: - ... - - @property - def lineno(self) -> int: - ... - - @property - def col_offset(self) -> int: - ... - - @property - def end_lineno(self) -> int: - ... - - @property - def end_col_offset(self) -> int: - ... - - @runtime_checkable class HeaderAndBodyBlock(Protocol): header: Any @@ -108,302 +22,3 @@ class HeaderAndBodyBlock(Protocol): @runtime_checkable class BodyBlock(Protocol): body: List[Any] - - -def range_from_token(token: Token) -> Range: - return Range( - start=Position(line=token.lineno - 1, character=token.col_offset), - end=Position( - line=token.lineno - 1, - character=token.end_col_offset, - ), - ) - - -class FirstAndLastRealStatementFinder(async_ast.Visitor): - def __init__(self) -> None: - super().__init__() - self.first_statement: Optional[ast.AST] = None - self.last_statement: Optional[ast.AST] = None - - @classmethod - def find_from(cls, model: ast.AST) -> Tuple[Optional[ast.AST], Optional[ast.AST]]: - finder = cls() - finder.visit(model) - return finder.first_statement, finder.last_statement - - def visit_Statement(self, statement: ast.AST) -> None: # noqa: N802 - from robot.parsing.model.statements import EmptyLine - - if not isinstance(statement, EmptyLine): - if self.first_statement is None: - self.first_statement = statement - - self.last_statement = statement - - -def _get_non_data_range_from_node( - node: ast.AST, only_start: bool = False, allow_comments: bool = False -) -> Optional[Range]: - from robot.parsing.lexer import Token as RobotToken - - if isinstance(node, HasTokens) and node.tokens: - start_token = next( - ( - v - for v in node.tokens - if v.type - not in [ - RobotToken.SEPARATOR, - *([] if allow_comments else [RobotToken.COMMENT]), - RobotToken.CONTINUATION, - RobotToken.EOL, - RobotToken.EOS, - ] - ), - None, - ) - - if only_start and start_token is not None: - end_tokens: Sequence[Token] = [t for t in node.tokens if start_token.lineno == t.lineno] - else: - end_tokens = node.tokens - - end_token = next( - ( - v - for v in reversed(end_tokens) - if v.type - not in [ - RobotToken.SEPARATOR, - *([] if allow_comments else [RobotToken.COMMENT]), - RobotToken.CONTINUATION, - RobotToken.EOL, - RobotToken.EOS, - ] - ), - None, - ) - if start_token is not None and end_token is not None: - return Range(start=range_from_token(start_token).start, end=range_from_token(end_token).end) - return None - - -def range_from_node( - node: ast.AST, skip_non_data: bool = False, only_start: bool = False, allow_comments: bool = False -) -> Range: - if skip_non_data: - if isinstance(node, HasTokens) and node.tokens: - result = _get_non_data_range_from_node(node, only_start, allow_comments) - if result is not None: - return result - else: - first_stmt, last_stmt = FirstAndLastRealStatementFinder.find_from(node) - if first_stmt is not None: - first_range = _get_non_data_range_from_node(first_stmt, only_start, allow_comments) - if first_range is not None and last_stmt is not None: - last_range = _get_non_data_range_from_node(last_stmt, only_start, allow_comments) - if last_range is not None: - return Range(start=first_range.start, end=last_range.end) - - return Range( - start=Position(line=node.lineno - 1, character=node.col_offset), - end=Position( - line=node.end_lineno - 1 if node.end_lineno is not None else -1, - character=node.end_col_offset if node.end_col_offset is not None else -1, - ), - ) - - -def token_in_range(token: Token, range: Range, include_end: bool = False) -> bool: - token_range = range_from_token(token) - return token_range.start.is_in_range(range, include_end) or token_range.end.is_in_range(range, include_end) - - -def node_in_range(node: ast.AST, range: Range, include_end: bool = False) -> bool: - node_range = range_from_node(node) - return node_range.start.is_in_range(range, include_end) or node_range.end.is_in_range(range, include_end) - - -def range_from_node_or_token(node: Optional[ast.AST], token: Optional[Token]) -> Range: - if token is not None: - return range_from_token(token) - if node is not None: - return range_from_node(node, True) - return Range.zero() - - -def is_not_variable_token(token: Token) -> bool: - from robot.errors import VariableError - - try: - r = list(token.tokenize_variables()) - if len(r) == 1 and r[0] == token: - return True - except VariableError: - pass - return False - - -def whitespace_at_begin_of_token(token: Token) -> int: - s = str(token.value) - - result = 0 - for c in s: - if c == " ": - result += 1 - elif c == "\t": - result += 2 - else: - break - return result - - -def whitespace_from_begin_of_token(token: Token) -> str: - s = str(token.value) - - result = "" - for c in s: - if c in [" ", "\t"]: - result += c - else: - break - - return result - - -def get_tokens_at_position(node: HasTokens, position: Position, include_end: bool = False) -> List[Token]: - return [ - t - for t in node.tokens - if position.is_in_range(range := range_from_token(t), include_end) or include_end and range.end == position - ] - - -def iter_nodes_at_position(node: ast.AST, position: Position, include_end: bool = False) -> AsyncIterator[ast.AST]: - return ( - n - async for n in async_ast.iter_nodes(node) - if position.is_in_range(range := range_from_node(n), include_end) or include_end and range.end == position - ) - - -async def get_nodes_at_position(node: ast.AST, position: Position, include_end: bool = False) -> List[ast.AST]: - return [n async for n in iter_nodes_at_position(node, position, include_end)] - - -async def get_node_at_position(node: ast.AST, position: Position, include_end: bool = False) -> Optional[ast.AST]: - result_nodes = await get_nodes_at_position(node, position, include_end) - if not result_nodes: - return None - - return result_nodes[-1] - - -def _tokenize_no_variables(token: Token) -> Iterator[Token]: - yield token - - -def tokenize_variables( - token: Token, identifiers: str = "$@&%", ignore_errors: bool = False, *, extra_types: Optional[Set[str]] = None -) -> Iterator[Token]: - from robot.api.parsing import Token as RobotToken - - if get_robot_version() < (7, 0): - from robot.variables import VariableIterator - else: - from robot.variables import VariableMatches as VariableIterator - - if token.type not in { - *RobotToken.ALLOW_VARIABLES, - RobotToken.KEYWORD, - RobotToken.ASSIGN, - *(extra_types if extra_types is not None else set()), - }: - return _tokenize_no_variables(token) - - value = token.value - - variables = VariableIterator(value, identifiers=identifiers, ignore_errors=ignore_errors) - if not variables: - return _tokenize_no_variables(token) - if get_robot_version() < (7, 0): - return _tokenize_variables_old(token, variables) - return _tokenize_variables(token, variables) - - -def _tokenize_variables(token: Token, matches: Any) -> Iterator[Token]: - from robot.api.parsing import Token as RobotToken - - lineno = token.lineno - col_offset = token.col_offset - after = "" - for match in matches: - if match.before: - yield RobotToken(token.type, match.before, lineno, col_offset) - yield RobotToken(RobotToken.VARIABLE, match.match, lineno, col_offset + match.start) - col_offset += match.end - after = match.after - if after: - yield RobotToken(token.type, after, lineno, col_offset) - - -def _tokenize_variables_old(token: Token, variables: Any) -> Iterator[Token]: - from robot.api.parsing import Token as RobotToken - - lineno = token.lineno - col_offset = token.col_offset - remaining = "" - for before, variable, remaining in variables: - if before: - yield RobotToken(token.type, before, lineno, col_offset) - col_offset += len(before) - yield RobotToken(RobotToken.VARIABLE, variable, lineno, col_offset) - col_offset += len(variable) - if remaining: - yield RobotToken(token.type, remaining, lineno, col_offset) - - -def iter_over_keyword_names_and_owners(full_name: str) -> Iterator[Tuple[Optional[str], ...]]: - yield None, full_name - - tokens = full_name.split(".") - if len(tokens) > 1: - for i in range(1, len(tokens)): - yield ".".join(tokens[:i]), ".".join(tokens[i:]) - - -def strip_variable_token(token: Token) -> Token: - from robot.api.parsing import Token as RobotToken - - if ( - token.type == RobotToken.VARIABLE - and token.value[:1] in "$@&%" - and token.value[1:2] == "{" - and token.value[-1:] == "}" - ): - value = token.value[2:-1] - - stripped_value = value.lstrip() - stripped_offset = len(value) - len(stripped_value) - return cast( - Token, RobotToken(token.type, stripped_value.rstrip(), token.lineno, token.col_offset + 2 + stripped_offset) - ) - - return token - - -def get_variable_token(token: Token) -> Optional[Token]: - from robot.parsing.lexer.tokens import Token as RobotToken - - return next( - ( - v - for v in itertools.dropwhile( - lambda t: t.type in RobotToken.NON_DATA_TOKENS, - tokenize_variables(token, ignore_errors=True), - ) - if v.type == RobotToken.VARIABLE - ), - None, - ) diff --git a/packages/robot/src/robotcode/robot/utils/ast.py b/packages/robot/src/robotcode/robot/utils/ast.py index 78c97ca37..a5fbc9be4 100644 --- a/packages/robot/src/robotcode/robot/utils/ast.py +++ b/packages/robot/src/robotcode/robot/utils/ast.py @@ -2,16 +2,7 @@ import ast import itertools -from typing import ( - Any, - AsyncIterator, - Iterator, - List, - Optional, - Sequence, - Set, - Tuple, -) +from typing import Any, Iterator, List, Optional, Sequence, Set, Tuple from robotcode.core.lsp.types import Position, Range @@ -206,20 +197,20 @@ def get_tokens_at_position(node: Statement, position: Position, include_end: boo ] -def iter_nodes_at_position(node: ast.AST, position: Position, include_end: bool = False) -> AsyncIterator[ast.AST]: +def iter_nodes_at_position(node: ast.AST, position: Position, include_end: bool = False) -> Iterator[ast.AST]: return ( n - async for n in visitors.iter_nodes(node) + for n in iter_nodes(node) if position.is_in_range(range := range_from_node(n), include_end) or include_end and range.end == position ) -async def get_nodes_at_position(node: ast.AST, position: Position, include_end: bool = False) -> List[ast.AST]: - return [n async for n in iter_nodes_at_position(node, position, include_end)] +def get_nodes_at_position(node: ast.AST, position: Position, include_end: bool = False) -> List[ast.AST]: + return [n for n in iter_nodes_at_position(node, position, include_end)] -async def get_node_at_position(node: ast.AST, position: Position, include_end: bool = False) -> Optional[ast.AST]: - result_nodes = await get_nodes_at_position(node, position, include_end) +def get_node_at_position(node: ast.AST, position: Position, include_end: bool = False) -> Optional[ast.AST]: + result_nodes = get_nodes_at_position(node, position, include_end) if not result_nodes: return None