From 9431901d3a40781fc704b4550692410fcfa70aef Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Mon, 13 Jan 2025 12:07:37 -0500 Subject: [PATCH 01/22] Move task to new location --- .github/CODEOWNERS | 2 +- src/sentry/api/endpoints/organization_derive_code_mappings.py | 2 +- .../{derive_code_mappings.py => auto_source_code_configs.py} | 0 ...derive_code_mappings.py => test_auto_source_code_configs.py} | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename src/sentry/tasks/{derive_code_mappings.py => auto_source_code_configs.py} (100%) rename tests/sentry/tasks/{test_derive_code_mappings.py => test_auto_source_code_configs.py} (99%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2ea484683b0423..16338f5b2c951d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -539,6 +539,7 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py @ge /src/sentry/tasks/auto_ongoing_issues.py @getsentry/issues /src/sentry/tasks/auto_remove_inbox.py @getsentry/issues /src/sentry/tasks/auto_resolve_issues.py @getsentry/issues +/src/sentry/tasks/auto_source_code_configs.py @getsentry/issues /src/sentry/tasks/check_new_issue_threshold_met.py @getsentry/issues /src/sentry/tasks/clear_expired_resolutions.py @getsentry/issues /src/sentry/tasks/clear_expired_rulesnoozes.py @getsentry/issues @@ -547,7 +548,6 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py @ge /src/sentry/tasks/commit_context.py @getsentry/issues /src/sentry/tasks/commits.py @getsentry/issues /src/sentry/tasks/delete_seer_grouping_records.py @getsentry/issues -/src/sentry/tasks/derive_code_mappings.py @getsentry/issues /src/sentry/tasks/embeddings_grouping/ @getsentry/issues /src/sentry/tasks/groupowner.py @getsentry/issues /src/sentry/tasks/merge.py @getsentry/issues diff --git a/src/sentry/api/endpoints/organization_derive_code_mappings.py b/src/sentry/api/endpoints/organization_derive_code_mappings.py index 485dec0e3f3755..1146a0ed9c49d6 100644 --- a/src/sentry/api/endpoints/organization_derive_code_mappings.py +++ b/src/sentry/api/endpoints/organization_derive_code_mappings.py @@ -21,7 +21,7 @@ ) from sentry.models.organization import Organization from sentry.models.project import Project -from sentry.tasks.derive_code_mappings import get_installation +from sentry.tasks.auto_source_code_configs import get_installation @region_silo_endpoint diff --git a/src/sentry/tasks/derive_code_mappings.py b/src/sentry/tasks/auto_source_code_configs.py similarity index 100% rename from src/sentry/tasks/derive_code_mappings.py rename to src/sentry/tasks/auto_source_code_configs.py diff --git a/tests/sentry/tasks/test_derive_code_mappings.py b/tests/sentry/tasks/test_auto_source_code_configs.py similarity index 99% rename from tests/sentry/tasks/test_derive_code_mappings.py rename to tests/sentry/tasks/test_auto_source_code_configs.py index cc593f0ea996be..61f547d8a6c02b 100644 --- a/tests/sentry/tasks/test_derive_code_mappings.py +++ b/tests/sentry/tasks/test_auto_source_code_configs.py @@ -13,7 +13,7 @@ from sentry.models.organization import OrganizationStatus from sentry.models.repository import Repository from sentry.shared_integrations.exceptions import ApiError -from sentry.tasks.derive_code_mappings import ( +from sentry.tasks.auto_source_code_configs import ( DeriveCodeMappingsErrorReason, derive_code_mappings, identify_stacktrace_paths, From ceeee900c16c1b6e7fe4ad2b5040bc1c45270620 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Mon, 13 Jan 2025 12:34:01 -0500 Subject: [PATCH 02/22] Refactor get_organization_installation --- .../organization_derive_code_mappings.py | 7 ++-- .../auto_source_code_config/integrations.py | 30 ++++++++++++++++ src/sentry/tasks/auto_source_code_configs.py | 35 +++---------------- .../tasks/test_auto_source_code_configs.py | 31 +++------------- 4 files changed, 43 insertions(+), 60 deletions(-) create mode 100644 src/sentry/issues/auto_source_code_config/integrations.py diff --git a/src/sentry/api/endpoints/organization_derive_code_mappings.py b/src/sentry/api/endpoints/organization_derive_code_mappings.py index 1146a0ed9c49d6..4b382be441c81e 100644 --- a/src/sentry/api/endpoints/organization_derive_code_mappings.py +++ b/src/sentry/api/endpoints/organization_derive_code_mappings.py @@ -19,9 +19,9 @@ Repo, create_code_mapping, ) +from sentry.issues.auto_source_code_config.integrations import get_organization_installation from sentry.models.organization import Organization from sentry.models.project import Project -from sentry.tasks.auto_source_code_configs import get_installation @region_silo_endpoint @@ -46,7 +46,8 @@ def get(self, request: Request, organization: Organization) -> Response: return Response(status=status.HTTP_403_FORBIDDEN) stacktrace_filename = request.GET.get("stacktraceFilename") - installation, _ = get_installation(organization) # only returns GitHub integrations + # It only returns the first GitHub integration + installation, _ = get_organization_installation(organization) if not installation: return self.respond( {"text": "Could not find this integration installed on your organization"}, @@ -88,7 +89,7 @@ def post(self, request: Request, organization: Organization) -> Response: if not features.has("organizations:derive-code-mappings", organization): return Response(status=status.HTTP_403_FORBIDDEN) - installation, organization_integration = get_installation(organization) + installation, organization_integration = get_organization_installation(organization) if not installation or not organization_integration: return self.respond( {"text": "Could not find this integration installed on your organization"}, diff --git a/src/sentry/issues/auto_source_code_config/integrations.py b/src/sentry/issues/auto_source_code_config/integrations.py new file mode 100644 index 00000000000000..3ca1bd2631bdad --- /dev/null +++ b/src/sentry/issues/auto_source_code_config/integrations.py @@ -0,0 +1,30 @@ +from sentry.constants import ObjectStatus +from sentry.integrations.base import IntegrationInstallation +from sentry.integrations.services.integration import RpcOrganizationIntegration, integration_service +from sentry.models.organization import Organization + +SUPPORTED_PROVIDERS = ["github"] + + +def get_organization_installation( + organization: Organization, +) -> tuple[IntegrationInstallation | None, RpcOrganizationIntegration | None]: + integrations = integration_service.get_integrations( + organization_id=organization.id, + providers=SUPPORTED_PROVIDERS, + status=ObjectStatus.ACTIVE, + ) + if len(integrations) == 0: + return None, None + + # XXX: We only operate on the first github integration for an organization. + integration = integrations[0] + organization_integration = integration_service.get_organization_integration( + integration_id=integration.id, organization_id=organization.id + ) + if not organization_integration: + return None, None + + installation = integration.get_installation(organization_id=organization.id) + + return installation, organization_integration diff --git a/src/sentry/tasks/auto_source_code_configs.py b/src/sentry/tasks/auto_source_code_configs.py index ca79f41e311a65..a1132ae0b01c44 100644 --- a/src/sentry/tasks/auto_source_code_configs.py +++ b/src/sentry/tasks/auto_source_code_configs.py @@ -3,21 +3,21 @@ import logging from collections.abc import Mapping from enum import StrEnum -from typing import TYPE_CHECKING, Any +from typing import Any from sentry_sdk import set_tag, set_user from sentry import features -from sentry.constants import ObjectStatus from sentry.db.models.fields.node import NodeData from sentry.integrations.github.integration import GitHubIntegration from sentry.integrations.models.repository_project_path_config import RepositoryProjectPathConfig -from sentry.integrations.services.integration import RpcOrganizationIntegration, integration_service +from sentry.integrations.services.integration import RpcOrganizationIntegration from sentry.integrations.source_code_management.metrics import ( SCMIntegrationInteractionEvent, SCMIntegrationInteractionType, ) from sentry.integrations.utils.code_mapping import CodeMapping, CodeMappingTreesHelper +from sentry.issues.auto_source_code_config.integrations import get_organization_installation from sentry.locks import locks from sentry.models.organization import Organization from sentry.models.project import Project @@ -31,9 +31,6 @@ logger = logging.getLogger(__name__) -if TYPE_CHECKING: - from sentry.integrations.base import IntegrationInstallation - class DeriveCodeMappingsErrorReason(StrEnum): UNEXPECTED_ERROR = "Unexpected error type while calling `get_trees_for_org()`." @@ -118,7 +115,7 @@ def derive_code_mappings( logger.info("No stacktrace paths found.", extra=extra) return - installation, organization_integration = get_installation(org) + installation, organization_integration = get_organization_installation(org) if not installation or not organization_integration: logger.info("No installation or organization integration found.", extra=extra) return @@ -189,30 +186,6 @@ def get_stacktrace(data: NodeData) -> list[Mapping[str, Any]]: return [] -def get_installation( - organization: Organization, -) -> tuple[IntegrationInstallation | None, RpcOrganizationIntegration | None]: - integrations = integration_service.get_integrations( - organization_id=organization.id, - providers=["github"], - status=ObjectStatus.ACTIVE, - ) - if len(integrations) == 0: - return None, None - - # XXX: We only operate on the first github integration for an organization. - integration = integrations[0] - organization_integration = integration_service.get_organization_integration( - integration_id=integration.id, organization_id=organization.id - ) - if not organization_integration: - return None, None - - installation = integration.get_installation(organization_id=organization.id) - - return installation, organization_integration - - def set_project_codemappings( code_mappings: list[CodeMapping], organization_integration: RpcOrganizationIntegration, diff --git a/tests/sentry/tasks/test_auto_source_code_configs.py b/tests/sentry/tasks/test_auto_source_code_configs.py index 61f547d8a6c02b..8c1a33bd099689 100644 --- a/tests/sentry/tasks/test_auto_source_code_configs.py +++ b/tests/sentry/tasks/test_auto_source_code_configs.py @@ -20,7 +20,6 @@ ) from sentry.testutils.asserts import assert_failure_metric, assert_halt_metric from sentry.testutils.cases import TestCase -from sentry.testutils.helpers import with_feature from sentry.testutils.silo import assume_test_silo_mode_of from sentry.testutils.skips import requires_snuba from sentry.utils.locking import UnableToAcquireLock @@ -70,7 +69,7 @@ def test_does_not_raise_installation_removed(self, mock_record): assert derive_code_mappings(self.project.id, self.event_data) is None assert_halt_metric(mock_record, error) - @patch("sentry.tasks.derive_code_mappings.logger") + @patch("sentry.tasks.auto_source_code_configs.logger") def test_raises_other_api_errors(self, mock_logger, mock_record): with patch( "sentry.integrations.github.client.GitHubBaseClient.get_trees_for_org", @@ -90,7 +89,7 @@ def test_unable_to_get_lock(self, mock_record): assert not RepositoryProjectPathConfig.objects.exists() assert_failure_metric(mock_record, error) - @patch("sentry.tasks.derive_code_mappings.logger") + @patch("sentry.tasks.auto_source_code_configs.logger") def test_raises_generic_errors(self, mock_logger, mock_record): with patch( "sentry.integrations.github.client.GitHubBaseClient.get_trees_for_org", @@ -651,26 +650,6 @@ def test_handle_duplicate_filenames_in_stacktrace(self): "sentry/tasks.py", ] - @with_feature({"organizations:derive-code-mappings": False}) - def test_feature_off(self): - event = self.store_event(data=self.test_data, project_id=self.project.id) - - assert not RepositoryProjectPathConfig.objects.filter(project_id=self.project.id).exists() - - with ( - patch( - "sentry.tasks.derive_code_mappings.identify_stacktrace_paths", - return_value={ - self.project: ["sentry/models/release.py", "sentry/tasks.py"], - }, - ) as mock_identify_stacktraces, - self.tasks(), - ): - derive_code_mappings(self.project.id, event.data) - - assert mock_identify_stacktraces.call_count == 0 - assert not RepositoryProjectPathConfig.objects.filter(project_id=self.project.id).exists() - @patch("sentry.integrations.github.integration.GitHubIntegration.get_trees_for_org") @patch( "sentry.integrations.utils.code_mapping.CodeMappingTreesHelper.generate_code_mappings", @@ -691,7 +670,7 @@ def test_derive_code_mappings_single_project( with ( patch( - "sentry.tasks.derive_code_mappings.identify_stacktrace_paths", + "sentry.tasks.auto_source_code_configs.identify_stacktrace_paths", return_value=["sentry/models/release.py", "sentry/tasks.py"], ) as mock_identify_stacktraces, self.tasks(), @@ -720,7 +699,7 @@ def test_skips_not_supported_platforms(self): ) ], ) - @patch("sentry.tasks.derive_code_mappings.logger") + @patch("sentry.tasks.auto_source_code_configs.logger") def test_derive_code_mappings_duplicates( self, mock_logger, mock_generate_code_mappings, mock_get_trees_for_org ): @@ -748,7 +727,7 @@ def test_derive_code_mappings_duplicates( with ( patch( - "sentry.tasks.derive_code_mappings.identify_stacktrace_paths", + "sentry.tasks.auto_source_code_configs.identify_stacktrace_paths", return_value=["sentry/models/release.py", "sentry/tasks.py"], ) as mock_identify_stacktraces, self.tasks(), From d104ce609dce3eeb17e9ced5c98097cdb2443ea1 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Mon, 13 Jan 2025 12:37:59 -0500 Subject: [PATCH 03/22] Move code_mapping file --- src/sentry/api/endpoints/group_ai_autofix.py | 2 +- src/sentry/api/endpoints/group_autofix_setup_check.py | 2 +- src/sentry/api/endpoints/organization_derive_code_mappings.py | 2 +- src/sentry/api/endpoints/project_repo_path_parsing.py | 2 +- src/sentry/api/endpoints/project_stacktrace_coverage.py | 2 +- src/sentry/autofix/utils.py | 2 +- src/sentry/integrations/github/client.py | 2 +- src/sentry/integrations/github/integration.py | 2 +- src/sentry/integrations/utils/commit_context.py | 4 +++- src/sentry/integrations/utils/stacktrace_link.py | 4 +++- .../utils => issues/auto_source_code_config}/code_mapping.py | 0 src/sentry/issues/endpoints/project_stacktrace_link.py | 2 +- src/sentry/tasks/auto_source_code_configs.py | 2 +- src/sentry/tasks/commit_context.py | 2 +- tests/sentry/integrations/github/test_integration.py | 2 +- tests/sentry/integrations/utils/test_code_mapping.py | 2 +- tests/sentry/tasks/test_auto_source_code_configs.py | 2 +- 17 files changed, 20 insertions(+), 16 deletions(-) rename src/sentry/{integrations/utils => issues/auto_source_code_config}/code_mapping.py (100%) diff --git a/src/sentry/api/endpoints/group_ai_autofix.py b/src/sentry/api/endpoints/group_ai_autofix.py index 7e5b6855cd1f5d..7b056f0e30cde7 100644 --- a/src/sentry/api/endpoints/group_ai_autofix.py +++ b/src/sentry/api/endpoints/group_ai_autofix.py @@ -17,7 +17,7 @@ from sentry.api.bases.group import GroupEndpoint from sentry.api.serializers import EventSerializer, serialize from sentry.autofix.utils import get_autofix_repos_from_project_code_mappings, get_autofix_state -from sentry.integrations.utils.code_mapping import get_sorted_code_mapping_configs +from sentry.issues.auto_source_code_config.code_mapping import get_sorted_code_mapping_configs from sentry.models.group import Group from sentry.seer.signed_seer_api import get_seer_salted_url, sign_with_seer_secret from sentry.tasks.autofix import check_autofix_status diff --git a/src/sentry/api/endpoints/group_autofix_setup_check.py b/src/sentry/api/endpoints/group_autofix_setup_check.py index 8cb0167f8fc238..70ead283887a20 100644 --- a/src/sentry/api/endpoints/group_autofix_setup_check.py +++ b/src/sentry/api/endpoints/group_autofix_setup_check.py @@ -14,7 +14,7 @@ from sentry.autofix.utils import get_autofix_repos_from_project_code_mappings from sentry.constants import ObjectStatus from sentry.integrations.services.integration import integration_service -from sentry.integrations.utils.code_mapping import get_sorted_code_mapping_configs +from sentry.issues.auto_source_code_config.code_mapping import get_sorted_code_mapping_configs from sentry.models.group import Group from sentry.models.organization import Organization from sentry.models.project import Project diff --git a/src/sentry/api/endpoints/organization_derive_code_mappings.py b/src/sentry/api/endpoints/organization_derive_code_mappings.py index 4b382be441c81e..9453a89ddc7820 100644 --- a/src/sentry/api/endpoints/organization_derive_code_mappings.py +++ b/src/sentry/api/endpoints/organization_derive_code_mappings.py @@ -12,7 +12,7 @@ ) from sentry.api.serializers import serialize from sentry.integrations.github.integration import GitHubIntegration -from sentry.integrations.utils.code_mapping import ( +from sentry.issues.auto_source_code_config.code_mapping import ( CodeMapping, CodeMappingTreesHelper, FrameFilename, diff --git a/src/sentry/api/endpoints/project_repo_path_parsing.py b/src/sentry/api/endpoints/project_repo_path_parsing.py index baa5bb573a7e3d..5959e296408ed6 100644 --- a/src/sentry/api/endpoints/project_repo_path_parsing.py +++ b/src/sentry/api/endpoints/project_repo_path_parsing.py @@ -12,7 +12,7 @@ from sentry.integrations.base import IntegrationFeatures from sentry.integrations.manager import default_manager as integrations from sentry.integrations.services.integration import RpcIntegration, integration_service -from sentry.integrations.utils.code_mapping import find_roots +from sentry.issues.auto_source_code_config.code_mapping import find_roots from sentry.models.repository import Repository diff --git a/src/sentry/api/endpoints/project_stacktrace_coverage.py b/src/sentry/api/endpoints/project_stacktrace_coverage.py index df2a6a3f612166..223d9fe4345241 100644 --- a/src/sentry/api/endpoints/project_stacktrace_coverage.py +++ b/src/sentry/api/endpoints/project_stacktrace_coverage.py @@ -9,9 +9,9 @@ from sentry.api.base import region_silo_endpoint from sentry.api.bases.project import ProjectEndpoint from sentry.api.serializers import serialize -from sentry.integrations.utils.code_mapping import get_sorted_code_mapping_configs from sentry.integrations.utils.codecov import codecov_enabled, fetch_codecov_data from sentry.integrations.utils.stacktrace_link import get_stacktrace_config +from sentry.issues.auto_source_code_config.code_mapping import get_sorted_code_mapping_configs from sentry.issues.endpoints.project_stacktrace_link import generate_context from sentry.models.project import Project from sentry.utils import metrics diff --git a/src/sentry/autofix/utils.py b/src/sentry/autofix/utils.py index 4de15e4be3ca06..b8a016b7f1ae0e 100644 --- a/src/sentry/autofix/utils.py +++ b/src/sentry/autofix/utils.py @@ -7,7 +7,7 @@ from django.conf import settings from pydantic import BaseModel -from sentry.integrations.utils.code_mapping import get_sorted_code_mapping_configs +from sentry.issues.auto_source_code_config.code_mapping import get_sorted_code_mapping_configs from sentry.models.project import Project from sentry.models.repository import Repository from sentry.seer.signed_seer_api import get_seer_salted_url, sign_with_seer_secret diff --git a/src/sentry/integrations/github/client.py b/src/sentry/integrations/github/client.py index b1f831e9b42c8b..87e7f14e003fa7 100644 --- a/src/sentry/integrations/github/client.py +++ b/src/sentry/integrations/github/client.py @@ -26,7 +26,7 @@ ) from sentry.integrations.source_code_management.repository import RepositoryClient from sentry.integrations.types import EXTERNAL_PROVIDERS, ExternalProviders -from sentry.integrations.utils.code_mapping import ( +from sentry.issues.auto_source_code_config.code_mapping import ( MAX_CONNECTION_ERRORS, Repo, RepoTree, diff --git a/src/sentry/integrations/github/integration.py b/src/sentry/integrations/github/integration.py index 9e6a4c7db04596..9af46d0a2d3847 100644 --- a/src/sentry/integrations/github/integration.py +++ b/src/sentry/integrations/github/integration.py @@ -32,11 +32,11 @@ from sentry.integrations.source_code_management.commit_context import CommitContextIntegration from sentry.integrations.source_code_management.repository import RepositoryIntegration from sentry.integrations.tasks.migrate_repo import migrate_repo -from sentry.integrations.utils.code_mapping import RepoTree from sentry.integrations.utils.metrics import ( IntegrationPipelineViewEvent, IntegrationPipelineViewType, ) +from sentry.issues.auto_source_code_config.code_mapping import RepoTree from sentry.models.repository import Repository from sentry.organizations.absolute_url import generate_organization_url from sentry.organizations.services.organization import RpcOrganizationSummary, organization_service diff --git a/src/sentry/integrations/utils/commit_context.py b/src/sentry/integrations/utils/commit_context.py index bdd64acc18ef7a..4b8e83257e7c09 100644 --- a/src/sentry/integrations/utils/commit_context.py +++ b/src/sentry/integrations/utils/commit_context.py @@ -18,7 +18,9 @@ FileBlameInfo, SourceLineInfo, ) -from sentry.integrations.utils.code_mapping import convert_stacktrace_frame_path_to_source_path +from sentry.issues.auto_source_code_config.code_mapping import ( + convert_stacktrace_frame_path_to_source_path, +) from sentry.models.commit import Commit from sentry.models.commitauthor import CommitAuthor from sentry.shared_integrations.exceptions import ApiError diff --git a/src/sentry/integrations/utils/stacktrace_link.py b/src/sentry/integrations/utils/stacktrace_link.py index 5aff6b93e60c9e..1919f2c87565ed 100644 --- a/src/sentry/integrations/utils/stacktrace_link.py +++ b/src/sentry/integrations/utils/stacktrace_link.py @@ -7,7 +7,9 @@ from sentry.integrations.models.repository_project_path_config import RepositoryProjectPathConfig from sentry.integrations.services.integration import integration_service from sentry.integrations.source_code_management.repository import RepositoryIntegration -from sentry.integrations.utils.code_mapping import convert_stacktrace_frame_path_to_source_path +from sentry.issues.auto_source_code_config.code_mapping import ( + convert_stacktrace_frame_path_to_source_path, +) from sentry.models.repository import Repository from sentry.shared_integrations.exceptions import ApiError from sentry.utils.event_frames import EventFrame diff --git a/src/sentry/integrations/utils/code_mapping.py b/src/sentry/issues/auto_source_code_config/code_mapping.py similarity index 100% rename from src/sentry/integrations/utils/code_mapping.py rename to src/sentry/issues/auto_source_code_config/code_mapping.py diff --git a/src/sentry/issues/endpoints/project_stacktrace_link.py b/src/sentry/issues/endpoints/project_stacktrace_link.py index 41f326fc31454b..02cc7cf5e813e8 100644 --- a/src/sentry/issues/endpoints/project_stacktrace_link.py +++ b/src/sentry/issues/endpoints/project_stacktrace_link.py @@ -17,8 +17,8 @@ from sentry.integrations.api.serializers.models.integration import IntegrationSerializer from sentry.integrations.base import IntegrationFeatures from sentry.integrations.services.integration import integration_service -from sentry.integrations.utils.code_mapping import get_sorted_code_mapping_configs from sentry.integrations.utils.stacktrace_link import StacktraceLinkOutcome, get_stacktrace_config +from sentry.issues.auto_source_code_config.code_mapping import get_sorted_code_mapping_configs from sentry.models.project import Project logger = logging.getLogger(__name__) diff --git a/src/sentry/tasks/auto_source_code_configs.py b/src/sentry/tasks/auto_source_code_configs.py index a1132ae0b01c44..4e3a39de49aa37 100644 --- a/src/sentry/tasks/auto_source_code_configs.py +++ b/src/sentry/tasks/auto_source_code_configs.py @@ -16,7 +16,7 @@ SCMIntegrationInteractionEvent, SCMIntegrationInteractionType, ) -from sentry.integrations.utils.code_mapping import CodeMapping, CodeMappingTreesHelper +from sentry.issues.auto_source_code_config.code_mapping import CodeMapping, CodeMappingTreesHelper from sentry.issues.auto_source_code_config.integrations import get_organization_installation from sentry.locks import locks from sentry.models.organization import Organization diff --git a/src/sentry/tasks/commit_context.py b/src/sentry/tasks/commit_context.py index 75a8dc0de2084b..9207690c31ee74 100644 --- a/src/sentry/tasks/commit_context.py +++ b/src/sentry/tasks/commit_context.py @@ -13,11 +13,11 @@ from sentry import analytics from sentry.api.serializers.models.release import get_users_for_authors from sentry.integrations.source_code_management.commit_context import CommitContextIntegration -from sentry.integrations.utils.code_mapping import get_sorted_code_mapping_configs from sentry.integrations.utils.commit_context import ( find_commit_context_for_event_all_frames, get_or_create_commit_from_blame, ) +from sentry.issues.auto_source_code_config.code_mapping import get_sorted_code_mapping_configs from sentry.locks import locks from sentry.models.commit import Commit from sentry.models.commitauthor import CommitAuthor diff --git a/tests/sentry/integrations/github/test_integration.py b/tests/sentry/integrations/github/test_integration.py index c6ddd42c62203e..06852afaf884a3 100644 --- a/tests/sentry/integrations/github/test_integration.py +++ b/tests/sentry/integrations/github/test_integration.py @@ -30,7 +30,7 @@ FileBlameInfo, SourceLineInfo, ) -from sentry.integrations.utils.code_mapping import Repo, RepoTree +from sentry.issues.auto_source_code_config.code_mapping import Repo, RepoTree from sentry.models.project import Project from sentry.models.repository import Repository from sentry.organizations.absolute_url import generate_organization_url diff --git a/tests/sentry/integrations/utils/test_code_mapping.py b/tests/sentry/integrations/utils/test_code_mapping.py index ec0af598fdd1fc..298f88cffbd8a3 100644 --- a/tests/sentry/integrations/utils/test_code_mapping.py +++ b/tests/sentry/integrations/utils/test_code_mapping.py @@ -3,7 +3,7 @@ import pytest from sentry.integrations.models.organization_integration import OrganizationIntegration -from sentry.integrations.utils.code_mapping import ( +from sentry.issues.auto_source_code_config.code_mapping import ( CodeMapping, CodeMappingTreesHelper, FrameFilename, diff --git a/tests/sentry/tasks/test_auto_source_code_configs.py b/tests/sentry/tasks/test_auto_source_code_configs.py index 8c1a33bd099689..5bbef9105b8232 100644 --- a/tests/sentry/tasks/test_auto_source_code_configs.py +++ b/tests/sentry/tasks/test_auto_source_code_configs.py @@ -9,7 +9,7 @@ from sentry.db.models.fields.node import NodeData from sentry.integrations.models.organization_integration import OrganizationIntegration from sentry.integrations.models.repository_project_path_config import RepositoryProjectPathConfig -from sentry.integrations.utils.code_mapping import CodeMapping, Repo, RepoTree +from sentry.issues.auto_source_code_config.code_mapping import CodeMapping, Repo, RepoTree from sentry.models.organization import OrganizationStatus from sentry.models.repository import Repository from sentry.shared_integrations.exceptions import ApiError From 2d8d091c5fbbe9b28b52e52cb96fc16e76df3b61 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Mon, 13 Jan 2025 12:52:58 -0500 Subject: [PATCH 04/22] Fix a bunch of renaming and moving files --- .github/CODEOWNERS | 2 +- src/sentry/api/urls.py | 2 +- src/sentry/conf/server.py | 3 ++- .../issues/auto_source_code_config/code_mapping.py | 2 ++ src/sentry/issues/endpoints/__init__.py | 2 ++ .../endpoints/organization_derive_code_mappings.py | 5 +++++ src/sentry/tasks/auto_source_code_configs.py | 12 +++++++----- src/sentry/tasks/post_process.py | 3 ++- src/sentry/utils/sdk.py | 3 +-- .../test_organization_derive_code_mappings.py | 0 .../integrations/github/tasks/test_pr_comment.py | 4 ++-- tests/sentry/tasks/test_post_process.py | 12 ++++++------ 12 files changed, 31 insertions(+), 19 deletions(-) rename src/sentry/{api => issues}/endpoints/organization_derive_code_mappings.py (96%) rename tests/sentry/api/endpoints/{ => issues}/test_organization_derive_code_mappings.py (100%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 16338f5b2c951d..4c29d2a5dcf70a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -574,6 +574,7 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py @ge /tests/sentry/tasks/test_auto_ongoing_issues.py @getsentry/issues /tests/sentry/tasks/test_auto_remove_inbox.py @getsentry/issues /tests/sentry/tasks/test_auto_resolve_issues.py @getsentry/issues +/tests/sentry/tasks/test_auto_source_code_configs.py @getsentry/issues /tests/sentry/tasks/test_backfill_seer_grouping_records.py @getsentry/issues /tests/sentry/tasks/test_check_new_issue_threshold_met.py @getsentry/issues /tests/sentry/tasks/test_clear_expired_resolutions.py @getsentry/issues @@ -583,7 +584,6 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py @ge /tests/sentry/tasks/test_commit_context.py @getsentry/issues /tests/sentry/tasks/test_commits.py @getsentry/issues /tests/sentry/tasks/test_delete_seer_grouping_records.py @getsentry/issues -/tests/sentry/tasks/test_derive_code_mappings.py @getsentry/issues /tests/sentry/tasks/test_groupowner.py @getsentry/issues /tests/sentry/tasks/test_merge.py @getsentry/issues /tests/sentry/tasks/test_post_process.py @getsentry/issues diff --git a/src/sentry/api/urls.py b/src/sentry/api/urls.py index 1e5074c0f6e44f..eef87589a2a919 100644 --- a/src/sentry/api/urls.py +++ b/src/sentry/api/urls.py @@ -177,6 +177,7 @@ GroupSimilarIssuesEndpoint, GroupTombstoneDetailsEndpoint, GroupTombstoneEndpoint, + OrganizationDeriveCodeMappingsEndpoint, OrganizationGroupIndexEndpoint, OrganizationGroupIndexStatsEndpoint, OrganizationGroupSearchViewsEndpoint, @@ -447,7 +448,6 @@ OrganizationDashboardWidgetDetailsEndpoint, ) from .endpoints.organization_dashboards import OrganizationDashboardsEndpoint -from .endpoints.organization_derive_code_mappings import OrganizationDeriveCodeMappingsEndpoint from .endpoints.organization_details import OrganizationDetailsEndpoint from .endpoints.organization_environments import OrganizationEnvironmentsEndpoint from .endpoints.organization_event_details import OrganizationEventDetailsEndpoint diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index c1e901f2a90ed0..5b40e376d8dbaf 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -818,7 +818,7 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str: "sentry.dynamic_sampling.tasks.sliding_window_org", "sentry.dynamic_sampling.tasks.utils", "sentry.dynamic_sampling.tasks.custom_rule_notifications", - "sentry.tasks.derive_code_mappings", + "sentry.tasks.auto_source_code_configs", "sentry.ingest.transaction_clusterer.tasks", "sentry.tasks.auto_enable_codecov", "sentry.tasks.weekly_escalating_forecast", @@ -980,6 +980,7 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str: Queue("replays.delete_replay", routing_key="replays.delete_replay"), Queue("counters-0", routing_key="counters-0"), Queue("triggers-0", routing_key="triggers-0"), + # XXX: To be renamed to auto_source_code_configs Queue("derive_code_mappings", routing_key="derive_code_mappings"), Queue("transactions.name_clusterer", routing_key="transactions.name_clusterer"), Queue("auto_enable_codecov", routing_key="auto_enable_codecov"), diff --git a/src/sentry/issues/auto_source_code_config/code_mapping.py b/src/sentry/issues/auto_source_code_config/code_mapping.py index f8d848dbc6a8e1..4c161c9a6a2c1a 100644 --- a/src/sentry/issues/auto_source_code_config/code_mapping.py +++ b/src/sentry/issues/auto_source_code_config/code_mapping.py @@ -13,6 +13,8 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) +SUPPORTED_LANGUAGES = ["javascript", "python", "node", "ruby", "php", "go", "csharp"] + SLASH = "/" BACKSLASH = "\\" # This is the Python representation of a single backslash diff --git a/src/sentry/issues/endpoints/__init__.py b/src/sentry/issues/endpoints/__init__.py index a1f812f708f8f0..45e3438ff6e869 100644 --- a/src/sentry/issues/endpoints/__init__.py +++ b/src/sentry/issues/endpoints/__init__.py @@ -10,6 +10,7 @@ from .group_similar_issues_embeddings import GroupSimilarIssuesEmbeddingsEndpoint from .group_tombstone import GroupTombstoneEndpoint from .group_tombstone_details import GroupTombstoneDetailsEndpoint +from .organization_derive_code_mappings import OrganizationDeriveCodeMappingsEndpoint from .organization_eventid import EventIdLookupEndpoint from .organization_group_index import OrganizationGroupIndexEndpoint from .organization_group_index_stats import OrganizationGroupIndexStatsEndpoint @@ -42,6 +43,7 @@ "GroupSimilarIssuesEndpoint", "GroupTombstoneDetailsEndpoint", "GroupTombstoneEndpoint", + "OrganizationDeriveCodeMappingsEndpoint", "OrganizationGroupIndexEndpoint", "OrganizationGroupIndexStatsEndpoint", "OrganizationGroupSearchViewsEndpoint", diff --git a/src/sentry/api/endpoints/organization_derive_code_mappings.py b/src/sentry/issues/endpoints/organization_derive_code_mappings.py similarity index 96% rename from src/sentry/api/endpoints/organization_derive_code_mappings.py rename to src/sentry/issues/endpoints/organization_derive_code_mappings.py index 9453a89ddc7820..965120dfcf92de 100644 --- a/src/sentry/api/endpoints/organization_derive_code_mappings.py +++ b/src/sentry/issues/endpoints/organization_derive_code_mappings.py @@ -26,6 +26,11 @@ @region_silo_endpoint class OrganizationDeriveCodeMappingsEndpoint(OrganizationEndpoint): + """ + In the UI, we have a prompt to derive code mappings from the stacktrace filename. + This endpoint is used to get the possible code mappings for it. + """ + owner = ApiOwner.ISSUES publish_status = { "GET": ApiPublishStatus.UNKNOWN, diff --git a/src/sentry/tasks/auto_source_code_configs.py b/src/sentry/tasks/auto_source_code_configs.py index 4e3a39de49aa37..6f4a20f27444cc 100644 --- a/src/sentry/tasks/auto_source_code_configs.py +++ b/src/sentry/tasks/auto_source_code_configs.py @@ -16,7 +16,11 @@ SCMIntegrationInteractionEvent, SCMIntegrationInteractionType, ) -from sentry.issues.auto_source_code_config.code_mapping import CodeMapping, CodeMappingTreesHelper +from sentry.issues.auto_source_code_config.code_mapping import ( + SUPPORTED_LANGUAGES, + CodeMapping, + CodeMappingTreesHelper, +) from sentry.issues.auto_source_code_config.integrations import get_organization_installation from sentry.locks import locks from sentry.models.organization import Organization @@ -27,8 +31,6 @@ from sentry.utils.locking import UnableToAcquireLock from sentry.utils.safe import get_path -SUPPORTED_LANGUAGES = ["javascript", "python", "node", "ruby", "php", "go", "csharp"] - logger = logging.getLogger(__name__) @@ -79,8 +81,8 @@ def process_error(error: ApiError, extra: dict[str, str]) -> None: @instrumented_task( - name="sentry.tasks.derive_code_mappings.derive_code_mappings", - queue="derive_code_mappings", + name="sentry.tasks.auto_source_code_configs.derive_code_mappings", + queue="derive_code_mappings", # XXX: To be renamed to auto_source_code_configs default_retry_delay=60 * 10, max_retries=3, ) diff --git a/src/sentry/tasks/post_process.py b/src/sentry/tasks/post_process.py index e5bb774e6438df..fedfcd0cc140d9 100644 --- a/src/sentry/tasks/post_process.py +++ b/src/sentry/tasks/post_process.py @@ -994,7 +994,8 @@ def process_code_mappings(job: PostProcessJob) -> None: if job["is_reprocessed"]: return - from sentry.tasks.derive_code_mappings import SUPPORTED_LANGUAGES, derive_code_mappings + from sentry.issues.auto_source_code_config.code_mapping import SUPPORTED_LANGUAGES + from sentry.tasks.auto_source_code_configs import derive_code_mappings try: event = job["event"] diff --git a/src/sentry/utils/sdk.py b/src/sentry/utils/sdk.py index 6bcd6e13944e2c..c6b120955919ee 100644 --- a/src/sentry/utils/sdk.py +++ b/src/sentry/utils/sdk.py @@ -49,6 +49,7 @@ # Tasks not included here are sampled with `SENTRY_BACKEND_APM_SAMPLING`. # If a parent task schedules other tasks, rates propagate to the children. SAMPLED_TASKS = { + "sentry.tasks.auto_source_code_configs.derive_code_mappings": settings.SAMPLED_DEFAULT_RATE, "sentry.tasks.send_ping": settings.SAMPLED_DEFAULT_RATE, "sentry.tasks.store.process_event": settings.SENTRY_PROCESS_EVENT_APM_SAMPLING, "sentry.tasks.store.process_event_from_reprocessing": settings.SENTRY_PROCESS_EVENT_APM_SAMPLING, @@ -71,8 +72,6 @@ "sentry.tasks.summaries.weekly_reports.prepare_organization_report": 0.1 * settings.SENTRY_BACKEND_APM_SAMPLING, "sentry.profiles.task.process_profile": 0.1 * settings.SENTRY_BACKEND_APM_SAMPLING, - "sentry.tasks.derive_code_mappings.process_organizations": settings.SAMPLED_DEFAULT_RATE, - "sentry.tasks.derive_code_mappings.derive_code_mappings": settings.SAMPLED_DEFAULT_RATE, "sentry.monitors.tasks.clock_pulse": 1.0, "sentry.tasks.auto_enable_codecov": settings.SAMPLED_DEFAULT_RATE, "sentry.dynamic_sampling.tasks.boost_low_volume_projects": 1.0, diff --git a/tests/sentry/api/endpoints/test_organization_derive_code_mappings.py b/tests/sentry/api/endpoints/issues/test_organization_derive_code_mappings.py similarity index 100% rename from tests/sentry/api/endpoints/test_organization_derive_code_mappings.py rename to tests/sentry/api/endpoints/issues/test_organization_derive_code_mappings.py diff --git a/tests/sentry/integrations/github/tasks/test_pr_comment.py b/tests/sentry/integrations/github/tasks/test_pr_comment.py index ca53cac157ce31..d8e1a14da3d251 100644 --- a/tests/sentry/integrations/github/tasks/test_pr_comment.py +++ b/tests/sentry/integrations/github/tasks/test_pr_comment.py @@ -389,7 +389,7 @@ def test_format_comment(self): issues = [ PullRequestIssue( title="TypeError", - subtitle="sentry.tasks.derive_code_mappings.derive_code_mappings", + subtitle="sentry.tasks.auto_source_code_configs.derive_code_mappings", url="https://sentry.sentry.io/issues/", ), PullRequestIssue( @@ -400,7 +400,7 @@ def test_format_comment(self): ] formatted_comment = format_comment(issues) - expected_comment = "## Suspect Issues\nThis pull request was deployed and Sentry observed the following issues:\n\n- ‼️ **TypeError** `sentry.tasks.derive_code_mappings.derive_code_m...` [View Issue](https://sentry.sentry.io/issues/?referrer=github-pr-bot)\n- ‼️ **KafkaException** `query_subscription_consumer_process_message` [View Issue](https://sentry.sentry.io/stats/?referrer=github-pr-bot)\n\nDid you find this useful? React with a 👍 or 👎" + expected_comment = "## Suspect Issues\nThis pull request was deployed and Sentry observed the following issues:\n\n- ‼️ **TypeError** `sentry.tasks.auto_source_code_configs.derive_code_m...` [View Issue](https://sentry.sentry.io/issues/?referrer=github-pr-bot)\n- ‼️ **KafkaException** `query_subscription_consumer_process_message` [View Issue](https://sentry.sentry.io/stats/?referrer=github-pr-bot)\n\nDid you find this useful? React with a 👍 or 👎" assert formatted_comment == expected_comment diff --git a/tests/sentry/tasks/test_post_process.py b/tests/sentry/tasks/test_post_process.py index 7fe3d653ec7996..eef99af928aeb0 100644 --- a/tests/sentry/tasks/test_post_process.py +++ b/tests/sentry/tasks/test_post_process.py @@ -54,7 +54,7 @@ from sentry.rules.actions.base import EventAction from sentry.silo.base import SiloMode from sentry.silo.safety import unguarded_write -from sentry.tasks.derive_code_mappings import SUPPORTED_LANGUAGES +from sentry.tasks.auto_source_code_configs import SUPPORTED_LANGUAGES from sentry.tasks.merge import merge_groups from sentry.tasks.post_process import ( HIGHER_ISSUE_OWNERS_PER_PROJECT_PER_MIN_RATELIMIT, @@ -227,14 +227,14 @@ def _call_post_process_group(self, event: Event) -> None: event=event, ) - @patch("sentry.tasks.derive_code_mappings.derive_code_mappings") + @patch("sentry.tasks.auto_source_code_configs.derive_code_mappings") def test_derive_invalid_platform(self, mock_derive_code_mappings): event = self._create_event({"platform": "elixir"}) self._call_post_process_group(event) assert mock_derive_code_mappings.delay.call_count == 0 - @patch("sentry.tasks.derive_code_mappings.derive_code_mappings") + @patch("sentry.tasks.auto_source_code_configs.derive_code_mappings") def test_derive_supported_languages(self, mock_derive_code_mappings): for platform in SUPPORTED_LANGUAGES: event = self._create_event({"platform": platform}) @@ -242,7 +242,7 @@ def test_derive_supported_languages(self, mock_derive_code_mappings): assert mock_derive_code_mappings.delay.call_count == 1 - @patch("sentry.tasks.derive_code_mappings.derive_code_mappings") + @patch("sentry.tasks.auto_source_code_configs.derive_code_mappings") def test_only_maps_a_given_project_once_per_hour(self, mock_derive_code_mappings): dogs_project = self.create_project() maisey_event = self._create_event( @@ -287,7 +287,7 @@ def test_only_maps_a_given_project_once_per_hour(self, mock_derive_code_mappings self._call_post_process_group(bodhi_event) assert mock_derive_code_mappings.delay.call_count == 2 - @patch("sentry.tasks.derive_code_mappings.derive_code_mappings") + @patch("sentry.tasks.auto_source_code_configs.derive_code_mappings") def test_only_maps_a_given_issue_once_per_day(self, mock_derive_code_mappings): dogs_project = self.create_project() maisey_event1 = self._create_event( @@ -337,7 +337,7 @@ def test_only_maps_a_given_issue_once_per_day(self, mock_derive_code_mappings): self._call_post_process_group(maisey_event4) assert mock_derive_code_mappings.delay.call_count == 2 - @patch("sentry.tasks.derive_code_mappings.derive_code_mappings") + @patch("sentry.tasks.auto_source_code_configs.derive_code_mappings") def test_skipping_an_issue_doesnt_mark_it_processed(self, mock_derive_code_mappings): dogs_project = self.create_project() maisey_event = self._create_event( From af5248d3db91c6bb1e23533669bf9bd1be644d00 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Mon, 13 Jan 2025 13:00:24 -0500 Subject: [PATCH 05/22] Typing and minor fixing --- pyproject.toml | 2 ++ tests/sentry/tasks/test_post_process.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0608b167d20e33..a2d6f537fa497e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -415,6 +415,7 @@ module = [ "sentry.issues", "sentry.issues.analytics", "sentry.issues.apps", + "sentry.issues.auto_source_code_config.*", "sentry.issues.constants", "sentry.issues.endpoints", "sentry.issues.endpoints.actionable_items", @@ -426,6 +427,7 @@ module = [ "sentry.issues.endpoints.group_similar_issues_embeddings", "sentry.issues.endpoints.group_tombstone", "sentry.issues.endpoints.group_tombstone_details", + "sentry.issues.endpoints.organization_derive_code_mappings", "sentry.issues.endpoints.organization_eventid", "sentry.issues.endpoints.organization_group_index", "sentry.issues.endpoints.organization_group_index_stats", diff --git a/tests/sentry/tasks/test_post_process.py b/tests/sentry/tasks/test_post_process.py index eef99af928aeb0..12496199966010 100644 --- a/tests/sentry/tasks/test_post_process.py +++ b/tests/sentry/tasks/test_post_process.py @@ -21,6 +21,7 @@ from sentry.feedback.usecases.create_feedback import FeedbackCreationSource from sentry.integrations.models.integration import Integration from sentry.integrations.source_code_management.commit_context import CommitInfo, FileBlameInfo +from sentry.issues.auto_source_code_config.code_mapping import SUPPORTED_LANGUAGES from sentry.issues.grouptype import ( FeedbackGroup, GroupCategory, @@ -54,7 +55,6 @@ from sentry.rules.actions.base import EventAction from sentry.silo.base import SiloMode from sentry.silo.safety import unguarded_write -from sentry.tasks.auto_source_code_configs import SUPPORTED_LANGUAGES from sentry.tasks.merge import merge_groups from sentry.tasks.post_process import ( HIGHER_ISSUE_OWNERS_PER_PROJECT_PER_MIN_RATELIMIT, From 803be9d5bd7b4931d9ce7110f6d1714a8f38a350 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Mon, 13 Jan 2025 14:18:13 -0500 Subject: [PATCH 06/22] Minor typing --- src/sentry/issues/auto_source_code_config/code_mapping.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sentry/issues/auto_source_code_config/code_mapping.py b/src/sentry/issues/auto_source_code_config/code_mapping.py index 4c161c9a6a2c1a..1e1c0b487a284c 100644 --- a/src/sentry/issues/auto_source_code_config/code_mapping.py +++ b/src/sentry/issues/auto_source_code_config/code_mapping.py @@ -111,7 +111,9 @@ def __init__(self, frame_file_path: str) -> None: def __repr__(self) -> str: return f"FrameFilename: {self.full_path}" - def __eq__(self, other) -> bool: + def __eq__(self, other: object) -> bool: + if not isinstance(other, FrameFilename): + return False return self.full_path == other.full_path From 4d8f431c4a9255c90a2fa068b7de5505259974e1 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Tue, 14 Jan 2025 08:43:00 -0500 Subject: [PATCH 07/22] Fix patching --- tests/sentry/tasks/test_auto_source_code_configs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/sentry/tasks/test_auto_source_code_configs.py b/tests/sentry/tasks/test_auto_source_code_configs.py index 5bbef9105b8232..32b236561e859e 100644 --- a/tests/sentry/tasks/test_auto_source_code_configs.py +++ b/tests/sentry/tasks/test_auto_source_code_configs.py @@ -652,7 +652,7 @@ def test_handle_duplicate_filenames_in_stacktrace(self): @patch("sentry.integrations.github.integration.GitHubIntegration.get_trees_for_org") @patch( - "sentry.integrations.utils.code_mapping.CodeMappingTreesHelper.generate_code_mappings", + "sentry.issues.auto_source_code_config.code_mapping.CodeMappingTreesHelper.generate_code_mappings", return_value=[ CodeMapping( repo=Repo(name="repo", branch="master"), @@ -690,7 +690,7 @@ def test_skips_not_supported_platforms(self): @patch("sentry.integrations.github.integration.GitHubIntegration.get_trees_for_org") @patch( - "sentry.integrations.utils.code_mapping.CodeMappingTreesHelper.generate_code_mappings", + "sentry.issues.auto_source_code_config.code_mapping.CodeMappingTreesHelper.generate_code_mappings", return_value=[ CodeMapping( repo=Repo(name="repo", branch="master"), From 6c9471a77a493d20078ea4c110366e6aa224f538 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Tue, 14 Jan 2025 08:47:07 -0500 Subject: [PATCH 08/22] Fix test --- tests/sentry/integrations/github/tasks/test_pr_comment.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/sentry/integrations/github/tasks/test_pr_comment.py b/tests/sentry/integrations/github/tasks/test_pr_comment.py index d8e1a14da3d251..fb4a08a20799a2 100644 --- a/tests/sentry/integrations/github/tasks/test_pr_comment.py +++ b/tests/sentry/integrations/github/tasks/test_pr_comment.py @@ -400,7 +400,13 @@ def test_format_comment(self): ] formatted_comment = format_comment(issues) - expected_comment = "## Suspect Issues\nThis pull request was deployed and Sentry observed the following issues:\n\n- ‼️ **TypeError** `sentry.tasks.auto_source_code_configs.derive_code_m...` [View Issue](https://sentry.sentry.io/issues/?referrer=github-pr-bot)\n- ‼️ **KafkaException** `query_subscription_consumer_process_message` [View Issue](https://sentry.sentry.io/stats/?referrer=github-pr-bot)\n\nDid you find this useful? React with a 👍 or 👎" + expected_comment = """## Suspect Issues +This pull request was deployed and Sentry observed the following issues: + +- ‼️ **TypeError** `sentry.tasks.auto_source_code_configs.derive_co...` [View Issue](https://sentry.sentry.io/issues/?referrer=github-pr-bot) +- ‼️ **KafkaException** `query_subscription_consumer_process_message` [View Issue](https://sentry.sentry.io/stats/?referrer=github-pr-bot) + +Did you find this useful? React with a 👍 or 👎""" assert formatted_comment == expected_comment From fee1c2b835c6d316f22945f29c39eae33ae795a3 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Tue, 14 Jan 2025 08:53:48 -0500 Subject: [PATCH 09/22] Move test plus fix patching --- .../issues/test_organization_derive_code_mappings.py | 6 +++--- .../auto_source_code_config}/test_code_mapping.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) rename tests/sentry/{integrations/utils => issues/auto_source_code_config}/test_code_mapping.py (99%) diff --git a/tests/sentry/api/endpoints/issues/test_organization_derive_code_mappings.py b/tests/sentry/api/endpoints/issues/test_organization_derive_code_mappings.py index 5ddc739f3acdb8..483f1a57bf1fab 100644 --- a/tests/sentry/api/endpoints/issues/test_organization_derive_code_mappings.py +++ b/tests/sentry/api/endpoints/issues/test_organization_derive_code_mappings.py @@ -50,7 +50,7 @@ def test_get_single_match(self, mock_get_trees_for_org): } ] with patch( - "sentry.integrations.utils.code_mapping.CodeMappingTreesHelper.list_file_matches", + "sentry.issues.auto_source_code_config.CodeMappingTreesHelper.list_file_matches", return_value=expected_matches, ): response = self.client.get(self.url, data=config_data, format="json") @@ -72,7 +72,7 @@ def test_get_start_with_backslash(self, mock_get_trees_for_org): } ] with patch( - "sentry.integrations.utils.code_mapping.CodeMappingTreesHelper.list_file_matches", + "sentry.issues.auto_source_code_config.CodeMappingTreesHelper.list_file_matches", return_value=expected_matches, ): response = self.client.get(self.url, data=config_data, format="json") @@ -102,7 +102,7 @@ def test_get_multiple_matches(self, mock_get_trees_for_org): }, ] with patch( - "sentry.integrations.utils.code_mapping.CodeMappingTreesHelper.list_file_matches", + "sentry.issues.auto_source_code_config.CodeMappingTreesHelper.list_file_matches", return_value=expected_matches, ): response = self.client.get(self.url, data=config_data, format="json") diff --git a/tests/sentry/integrations/utils/test_code_mapping.py b/tests/sentry/issues/auto_source_code_config/test_code_mapping.py similarity index 99% rename from tests/sentry/integrations/utils/test_code_mapping.py rename to tests/sentry/issues/auto_source_code_config/test_code_mapping.py index 298f88cffbd8a3..d4216361c707b3 100644 --- a/tests/sentry/integrations/utils/test_code_mapping.py +++ b/tests/sentry/issues/auto_source_code_config/test_code_mapping.py @@ -192,7 +192,7 @@ def test_no_matches(self): code_mappings = self.code_mapping_helper.generate_code_mappings(stacktraces) assert code_mappings == [] - @patch("sentry.integrations.utils.code_mapping.logger") + @patch("sentry.issues.auto_source_code_config.code_mapping.logger") def test_matches_top_src_file(self, logger): stacktraces = ["setup.py"] code_mappings = self.code_mapping_helper.generate_code_mappings(stacktraces) @@ -246,7 +246,7 @@ def test_more_than_one_match_works_with_different_order(self): code_mappings = self.code_mapping_helper.generate_code_mappings(stacktraces) assert sorted(code_mappings) == sorted(self.expected_code_mappings) - @patch("sentry.integrations.utils.code_mapping.logger") + @patch("sentry.issues.auto_source_code_config.code_mapping.logger") def test_more_than_one_repo_match(self, logger): # XXX: There's a chance that we could infer package names but that is risky # repo 1: src/sentry/web/urls.py From 1e1829d9ad9f15bc540c444534b176371a5c4b31 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Tue, 14 Jan 2025 08:56:37 -0500 Subject: [PATCH 10/22] Include tests.sentry.issues as owned by our team --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4c29d2a5dcf70a..6be7c8e2532c86 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -530,6 +530,7 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py @ge /src/sentry/event_manager.py @getsentry/issues /src/sentry/eventstore/models.py @getsentry/issues /src/sentry/grouping/ @getsentry/issues +/src/sentry/issues/ @getsentry/issues /src/sentry/mediators/ @getsentry/issues /src/sentry/ratelimits/ @getsentry/issues /src/sentry/search/events/builder/issue_platform.py @getsentry/issues @@ -569,6 +570,7 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py @ge /tests/sentry/api/test_issue_search.py @getsentry/issues /tests/sentry/deletions/test_group.py @getsentry/issues /tests/sentry/event_manager/ @getsentry/issues +/tests/sentry/issues/ @getsentry/issues /tests/sentry/grouping/ @getsentry/issues /tests/sentry/search/ @getsentry/issues /tests/sentry/tasks/test_auto_ongoing_issues.py @getsentry/issues From a56e3a8e21ec132f317d40cbd3d5d2e583cadab9 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Tue, 14 Jan 2025 09:08:15 -0500 Subject: [PATCH 11/22] Make it all singular --- .github/CODEOWNERS | 4 ++-- src/sentry/conf/server.py | 5 +++-- src/sentry/tasks/auto_source_code_configs.py | 4 ++-- src/sentry/tasks/post_process.py | 2 +- src/sentry/utils/sdk.py | 2 +- .../integrations/github/tasks/test_pr_comment.py | 4 ++-- tests/sentry/tasks/test_auto_source_code_configs.py | 12 ++++++------ tests/sentry/tasks/test_post_process.py | 10 +++++----- 8 files changed, 22 insertions(+), 21 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6be7c8e2532c86..425258c27eb823 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -540,7 +540,7 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py @ge /src/sentry/tasks/auto_ongoing_issues.py @getsentry/issues /src/sentry/tasks/auto_remove_inbox.py @getsentry/issues /src/sentry/tasks/auto_resolve_issues.py @getsentry/issues -/src/sentry/tasks/auto_source_code_configs.py @getsentry/issues +/src/sentry/tasks/auto_source_code_config.py @getsentry/issues /src/sentry/tasks/check_new_issue_threshold_met.py @getsentry/issues /src/sentry/tasks/clear_expired_resolutions.py @getsentry/issues /src/sentry/tasks/clear_expired_rulesnoozes.py @getsentry/issues @@ -576,7 +576,7 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py @ge /tests/sentry/tasks/test_auto_ongoing_issues.py @getsentry/issues /tests/sentry/tasks/test_auto_remove_inbox.py @getsentry/issues /tests/sentry/tasks/test_auto_resolve_issues.py @getsentry/issues -/tests/sentry/tasks/test_auto_source_code_configs.py @getsentry/issues +/tests/sentry/tasks/test_auto_source_code_config.py @getsentry/issues /tests/sentry/tasks/test_backfill_seer_grouping_records.py @getsentry/issues /tests/sentry/tasks/test_check_new_issue_threshold_met.py @getsentry/issues /tests/sentry/tasks/test_clear_expired_resolutions.py @getsentry/issues diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index 5b40e376d8dbaf..d264a269239fc6 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -818,7 +818,7 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str: "sentry.dynamic_sampling.tasks.sliding_window_org", "sentry.dynamic_sampling.tasks.utils", "sentry.dynamic_sampling.tasks.custom_rule_notifications", - "sentry.tasks.auto_source_code_configs", + "sentry.tasks.auto_source_code_config", "sentry.ingest.transaction_clusterer.tasks", "sentry.tasks.auto_enable_codecov", "sentry.tasks.weekly_escalating_forecast", @@ -980,8 +980,9 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str: Queue("replays.delete_replay", routing_key="replays.delete_replay"), Queue("counters-0", routing_key="counters-0"), Queue("triggers-0", routing_key="triggers-0"), - # XXX: To be renamed to auto_source_code_configs + # XXX: Temporarilty keep in place until we have migrated to the new queue Queue("derive_code_mappings", routing_key="derive_code_mappings"), + Queue("auto_source_code_config", routing_key="auto_source_code_config"), Queue("transactions.name_clusterer", routing_key="transactions.name_clusterer"), Queue("auto_enable_codecov", routing_key="auto_enable_codecov"), Queue("weekly_escalating_forecast", routing_key="weekly_escalating_forecast"), diff --git a/src/sentry/tasks/auto_source_code_configs.py b/src/sentry/tasks/auto_source_code_configs.py index 6f4a20f27444cc..f96e480ea8b830 100644 --- a/src/sentry/tasks/auto_source_code_configs.py +++ b/src/sentry/tasks/auto_source_code_configs.py @@ -81,8 +81,8 @@ def process_error(error: ApiError, extra: dict[str, str]) -> None: @instrumented_task( - name="sentry.tasks.auto_source_code_configs.derive_code_mappings", - queue="derive_code_mappings", # XXX: To be renamed to auto_source_code_configs + name="sentry.tasks.auto_source_code_config.derive_code_mappings", + queue="derive_code_mappings", # XXX: To be renamed to auto_source_code_config default_retry_delay=60 * 10, max_retries=3, ) diff --git a/src/sentry/tasks/post_process.py b/src/sentry/tasks/post_process.py index fedfcd0cc140d9..000df1e8c3754f 100644 --- a/src/sentry/tasks/post_process.py +++ b/src/sentry/tasks/post_process.py @@ -995,7 +995,7 @@ def process_code_mappings(job: PostProcessJob) -> None: return from sentry.issues.auto_source_code_config.code_mapping import SUPPORTED_LANGUAGES - from sentry.tasks.auto_source_code_configs import derive_code_mappings + from sentry.tasks.auto_source_code_config import derive_code_mappings try: event = job["event"] diff --git a/src/sentry/utils/sdk.py b/src/sentry/utils/sdk.py index c6b120955919ee..0271aa88565093 100644 --- a/src/sentry/utils/sdk.py +++ b/src/sentry/utils/sdk.py @@ -49,7 +49,7 @@ # Tasks not included here are sampled with `SENTRY_BACKEND_APM_SAMPLING`. # If a parent task schedules other tasks, rates propagate to the children. SAMPLED_TASKS = { - "sentry.tasks.auto_source_code_configs.derive_code_mappings": settings.SAMPLED_DEFAULT_RATE, + "sentry.tasks.auto_source_code_config.derive_code_mappings": settings.SAMPLED_DEFAULT_RATE, "sentry.tasks.send_ping": settings.SAMPLED_DEFAULT_RATE, "sentry.tasks.store.process_event": settings.SENTRY_PROCESS_EVENT_APM_SAMPLING, "sentry.tasks.store.process_event_from_reprocessing": settings.SENTRY_PROCESS_EVENT_APM_SAMPLING, diff --git a/tests/sentry/integrations/github/tasks/test_pr_comment.py b/tests/sentry/integrations/github/tasks/test_pr_comment.py index fb4a08a20799a2..4cbb901678c21b 100644 --- a/tests/sentry/integrations/github/tasks/test_pr_comment.py +++ b/tests/sentry/integrations/github/tasks/test_pr_comment.py @@ -389,7 +389,7 @@ def test_format_comment(self): issues = [ PullRequestIssue( title="TypeError", - subtitle="sentry.tasks.auto_source_code_configs.derive_code_mappings", + subtitle="sentry.tasks.auto_source_code_config.derive_code_mappings", url="https://sentry.sentry.io/issues/", ), PullRequestIssue( @@ -403,7 +403,7 @@ def test_format_comment(self): expected_comment = """## Suspect Issues This pull request was deployed and Sentry observed the following issues: -- ‼️ **TypeError** `sentry.tasks.auto_source_code_configs.derive_co...` [View Issue](https://sentry.sentry.io/issues/?referrer=github-pr-bot) +- ‼️ **TypeError** `sentry.tasks.auto_source_code_config.derive_co...` [View Issue](https://sentry.sentry.io/issues/?referrer=github-pr-bot) - ‼️ **KafkaException** `query_subscription_consumer_process_message` [View Issue](https://sentry.sentry.io/stats/?referrer=github-pr-bot) Did you find this useful? React with a 👍 or 👎""" diff --git a/tests/sentry/tasks/test_auto_source_code_configs.py b/tests/sentry/tasks/test_auto_source_code_configs.py index 32b236561e859e..801af6228c1855 100644 --- a/tests/sentry/tasks/test_auto_source_code_configs.py +++ b/tests/sentry/tasks/test_auto_source_code_configs.py @@ -13,7 +13,7 @@ from sentry.models.organization import OrganizationStatus from sentry.models.repository import Repository from sentry.shared_integrations.exceptions import ApiError -from sentry.tasks.auto_source_code_configs import ( +from sentry.tasks.auto_source_code_config import ( DeriveCodeMappingsErrorReason, derive_code_mappings, identify_stacktrace_paths, @@ -69,7 +69,7 @@ def test_does_not_raise_installation_removed(self, mock_record): assert derive_code_mappings(self.project.id, self.event_data) is None assert_halt_metric(mock_record, error) - @patch("sentry.tasks.auto_source_code_configs.logger") + @patch("sentry.tasks.auto_source_code_config.logger") def test_raises_other_api_errors(self, mock_logger, mock_record): with patch( "sentry.integrations.github.client.GitHubBaseClient.get_trees_for_org", @@ -89,7 +89,7 @@ def test_unable_to_get_lock(self, mock_record): assert not RepositoryProjectPathConfig.objects.exists() assert_failure_metric(mock_record, error) - @patch("sentry.tasks.auto_source_code_configs.logger") + @patch("sentry.tasks.auto_source_code_config.logger") def test_raises_generic_errors(self, mock_logger, mock_record): with patch( "sentry.integrations.github.client.GitHubBaseClient.get_trees_for_org", @@ -670,7 +670,7 @@ def test_derive_code_mappings_single_project( with ( patch( - "sentry.tasks.auto_source_code_configs.identify_stacktrace_paths", + "sentry.tasks.auto_source_code_config.identify_stacktrace_paths", return_value=["sentry/models/release.py", "sentry/tasks.py"], ) as mock_identify_stacktraces, self.tasks(), @@ -699,7 +699,7 @@ def test_skips_not_supported_platforms(self): ) ], ) - @patch("sentry.tasks.auto_source_code_configs.logger") + @patch("sentry.tasks.auto_source_code_config.logger") def test_derive_code_mappings_duplicates( self, mock_logger, mock_generate_code_mappings, mock_get_trees_for_org ): @@ -727,7 +727,7 @@ def test_derive_code_mappings_duplicates( with ( patch( - "sentry.tasks.auto_source_code_configs.identify_stacktrace_paths", + "sentry.tasks.auto_source_code_config.identify_stacktrace_paths", return_value=["sentry/models/release.py", "sentry/tasks.py"], ) as mock_identify_stacktraces, self.tasks(), diff --git a/tests/sentry/tasks/test_post_process.py b/tests/sentry/tasks/test_post_process.py index 12496199966010..560f654d8f2d70 100644 --- a/tests/sentry/tasks/test_post_process.py +++ b/tests/sentry/tasks/test_post_process.py @@ -227,14 +227,14 @@ def _call_post_process_group(self, event: Event) -> None: event=event, ) - @patch("sentry.tasks.auto_source_code_configs.derive_code_mappings") + @patch("sentry.tasks.auto_source_code_config.derive_code_mappings") def test_derive_invalid_platform(self, mock_derive_code_mappings): event = self._create_event({"platform": "elixir"}) self._call_post_process_group(event) assert mock_derive_code_mappings.delay.call_count == 0 - @patch("sentry.tasks.auto_source_code_configs.derive_code_mappings") + @patch("sentry.tasks.auto_source_code_config.derive_code_mappings") def test_derive_supported_languages(self, mock_derive_code_mappings): for platform in SUPPORTED_LANGUAGES: event = self._create_event({"platform": platform}) @@ -242,7 +242,7 @@ def test_derive_supported_languages(self, mock_derive_code_mappings): assert mock_derive_code_mappings.delay.call_count == 1 - @patch("sentry.tasks.auto_source_code_configs.derive_code_mappings") + @patch("sentry.tasks.auto_source_code_config.derive_code_mappings") def test_only_maps_a_given_project_once_per_hour(self, mock_derive_code_mappings): dogs_project = self.create_project() maisey_event = self._create_event( @@ -287,7 +287,7 @@ def test_only_maps_a_given_project_once_per_hour(self, mock_derive_code_mappings self._call_post_process_group(bodhi_event) assert mock_derive_code_mappings.delay.call_count == 2 - @patch("sentry.tasks.auto_source_code_configs.derive_code_mappings") + @patch("sentry.tasks.auto_source_code_config.derive_code_mappings") def test_only_maps_a_given_issue_once_per_day(self, mock_derive_code_mappings): dogs_project = self.create_project() maisey_event1 = self._create_event( @@ -337,7 +337,7 @@ def test_only_maps_a_given_issue_once_per_day(self, mock_derive_code_mappings): self._call_post_process_group(maisey_event4) assert mock_derive_code_mappings.delay.call_count == 2 - @patch("sentry.tasks.auto_source_code_configs.derive_code_mappings") + @patch("sentry.tasks.auto_source_code_config.derive_code_mappings") def test_skipping_an_issue_doesnt_mark_it_processed(self, mock_derive_code_mappings): dogs_project = self.create_project() maisey_event = self._create_event( From f2ce5731e1a4d1a81b94c23ebe6691f17ac55dff Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Tue, 14 Jan 2025 09:22:00 -0500 Subject: [PATCH 12/22] Reduce changes --- .../auto_source_code_config/integrations.py | 30 ---------------- .../organization_derive_code_mappings.py | 6 ++-- ..._configs.py => auto_source_code_config.py} | 35 ++++++++++++++++--- ...igs.py => test_auto_source_code_config.py} | 0 4 files changed, 34 insertions(+), 37 deletions(-) delete mode 100644 src/sentry/issues/auto_source_code_config/integrations.py rename src/sentry/tasks/{auto_source_code_configs.py => auto_source_code_config.py} (88%) rename tests/sentry/tasks/{test_auto_source_code_configs.py => test_auto_source_code_config.py} (100%) diff --git a/src/sentry/issues/auto_source_code_config/integrations.py b/src/sentry/issues/auto_source_code_config/integrations.py deleted file mode 100644 index 3ca1bd2631bdad..00000000000000 --- a/src/sentry/issues/auto_source_code_config/integrations.py +++ /dev/null @@ -1,30 +0,0 @@ -from sentry.constants import ObjectStatus -from sentry.integrations.base import IntegrationInstallation -from sentry.integrations.services.integration import RpcOrganizationIntegration, integration_service -from sentry.models.organization import Organization - -SUPPORTED_PROVIDERS = ["github"] - - -def get_organization_installation( - organization: Organization, -) -> tuple[IntegrationInstallation | None, RpcOrganizationIntegration | None]: - integrations = integration_service.get_integrations( - organization_id=organization.id, - providers=SUPPORTED_PROVIDERS, - status=ObjectStatus.ACTIVE, - ) - if len(integrations) == 0: - return None, None - - # XXX: We only operate on the first github integration for an organization. - integration = integrations[0] - organization_integration = integration_service.get_organization_integration( - integration_id=integration.id, organization_id=organization.id - ) - if not organization_integration: - return None, None - - installation = integration.get_installation(organization_id=organization.id) - - return installation, organization_integration diff --git a/src/sentry/issues/endpoints/organization_derive_code_mappings.py b/src/sentry/issues/endpoints/organization_derive_code_mappings.py index 965120dfcf92de..504bcfa279309c 100644 --- a/src/sentry/issues/endpoints/organization_derive_code_mappings.py +++ b/src/sentry/issues/endpoints/organization_derive_code_mappings.py @@ -19,9 +19,9 @@ Repo, create_code_mapping, ) -from sentry.issues.auto_source_code_config.integrations import get_organization_installation from sentry.models.organization import Organization from sentry.models.project import Project +from sentry.tasks.auto_source_code_config import get_installation @region_silo_endpoint @@ -52,7 +52,7 @@ def get(self, request: Request, organization: Organization) -> Response: stacktrace_filename = request.GET.get("stacktraceFilename") # It only returns the first GitHub integration - installation, _ = get_organization_installation(organization) + installation, _ = get_installation(organization) if not installation: return self.respond( {"text": "Could not find this integration installed on your organization"}, @@ -94,7 +94,7 @@ def post(self, request: Request, organization: Organization) -> Response: if not features.has("organizations:derive-code-mappings", organization): return Response(status=status.HTTP_403_FORBIDDEN) - installation, organization_integration = get_organization_installation(organization) + installation, organization_integration = get_installation(organization) if not installation or not organization_integration: return self.respond( {"text": "Could not find this integration installed on your organization"}, diff --git a/src/sentry/tasks/auto_source_code_configs.py b/src/sentry/tasks/auto_source_code_config.py similarity index 88% rename from src/sentry/tasks/auto_source_code_configs.py rename to src/sentry/tasks/auto_source_code_config.py index f96e480ea8b830..5f8ac6c1e41594 100644 --- a/src/sentry/tasks/auto_source_code_configs.py +++ b/src/sentry/tasks/auto_source_code_config.py @@ -3,15 +3,16 @@ import logging from collections.abc import Mapping from enum import StrEnum -from typing import Any +from typing import TYPE_CHECKING, Any from sentry_sdk import set_tag, set_user from sentry import features +from sentry.constants import ObjectStatus from sentry.db.models.fields.node import NodeData from sentry.integrations.github.integration import GitHubIntegration from sentry.integrations.models.repository_project_path_config import RepositoryProjectPathConfig -from sentry.integrations.services.integration import RpcOrganizationIntegration +from sentry.integrations.services.integration import RpcOrganizationIntegration, integration_service from sentry.integrations.source_code_management.metrics import ( SCMIntegrationInteractionEvent, SCMIntegrationInteractionType, @@ -21,7 +22,6 @@ CodeMapping, CodeMappingTreesHelper, ) -from sentry.issues.auto_source_code_config.integrations import get_organization_installation from sentry.locks import locks from sentry.models.organization import Organization from sentry.models.project import Project @@ -33,6 +33,9 @@ logger = logging.getLogger(__name__) +if TYPE_CHECKING: + from sentry.integrations.base import IntegrationInstallation + class DeriveCodeMappingsErrorReason(StrEnum): UNEXPECTED_ERROR = "Unexpected error type while calling `get_trees_for_org()`." @@ -117,7 +120,7 @@ def derive_code_mappings( logger.info("No stacktrace paths found.", extra=extra) return - installation, organization_integration = get_organization_installation(org) + installation, organization_integration = get_installation(org) if not installation or not organization_integration: logger.info("No installation or organization integration found.", extra=extra) return @@ -188,6 +191,30 @@ def get_stacktrace(data: NodeData) -> list[Mapping[str, Any]]: return [] +def get_installation( + organization: Organization, +) -> tuple[IntegrationInstallation | None, RpcOrganizationIntegration | None]: + integrations = integration_service.get_integrations( + organization_id=organization.id, + providers=["github"], + status=ObjectStatus.ACTIVE, + ) + if len(integrations) == 0: + return None, None + + # XXX: We only operate on the first github integration for an organization. + integration = integrations[0] + organization_integration = integration_service.get_organization_integration( + integration_id=integration.id, organization_id=organization.id + ) + if not organization_integration: + return None, None + + installation = integration.get_installation(organization_id=organization.id) + + return installation, organization_integration + + def set_project_codemappings( code_mappings: list[CodeMapping], organization_integration: RpcOrganizationIntegration, diff --git a/tests/sentry/tasks/test_auto_source_code_configs.py b/tests/sentry/tasks/test_auto_source_code_config.py similarity index 100% rename from tests/sentry/tasks/test_auto_source_code_configs.py rename to tests/sentry/tasks/test_auto_source_code_config.py From 39bcaa3454b1c48676ccdd0d314b2eae690ac9e1 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Tue, 14 Jan 2025 09:42:17 -0500 Subject: [PATCH 13/22] Fix test --- .../issues/test_organization_derive_code_mappings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/sentry/api/endpoints/issues/test_organization_derive_code_mappings.py b/tests/sentry/api/endpoints/issues/test_organization_derive_code_mappings.py index 483f1a57bf1fab..8c196ba148d53d 100644 --- a/tests/sentry/api/endpoints/issues/test_organization_derive_code_mappings.py +++ b/tests/sentry/api/endpoints/issues/test_organization_derive_code_mappings.py @@ -50,7 +50,7 @@ def test_get_single_match(self, mock_get_trees_for_org): } ] with patch( - "sentry.issues.auto_source_code_config.CodeMappingTreesHelper.list_file_matches", + "sentry.issues.auto_source_code_config.code_mapping.CodeMappingTreesHelper.list_file_matches", return_value=expected_matches, ): response = self.client.get(self.url, data=config_data, format="json") @@ -72,7 +72,7 @@ def test_get_start_with_backslash(self, mock_get_trees_for_org): } ] with patch( - "sentry.issues.auto_source_code_config.CodeMappingTreesHelper.list_file_matches", + "sentry.issues.auto_source_code_config.code_mapping.CodeMappingTreesHelper.list_file_matches", return_value=expected_matches, ): response = self.client.get(self.url, data=config_data, format="json") From 8ccca3dabd15dd2ca2050af120e1a9e5fdfc1323 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Tue, 14 Jan 2025 10:02:53 -0500 Subject: [PATCH 14/22] Update codeowners --- .github/CODEOWNERS | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 425258c27eb823..15aa7382eeeff2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -519,7 +519,6 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py @ge ## Issues **/issues/** @getsentry/issues -*code_mappings*.py @getsentry/issues /src/sentry/api/helpers/actionable_items_helper.py @getsentry/issues /src/sentry/api/helpers/events.py @getsentry/issues /src/sentry/api/helpers/group_index/ @getsentry/issues @@ -530,7 +529,6 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py @ge /src/sentry/event_manager.py @getsentry/issues /src/sentry/eventstore/models.py @getsentry/issues /src/sentry/grouping/ @getsentry/issues -/src/sentry/issues/ @getsentry/issues /src/sentry/mediators/ @getsentry/issues /src/sentry/ratelimits/ @getsentry/issues /src/sentry/search/events/builder/issue_platform.py @getsentry/issues @@ -570,7 +568,6 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py @ge /tests/sentry/api/test_issue_search.py @getsentry/issues /tests/sentry/deletions/test_group.py @getsentry/issues /tests/sentry/event_manager/ @getsentry/issues -/tests/sentry/issues/ @getsentry/issues /tests/sentry/grouping/ @getsentry/issues /tests/sentry/search/ @getsentry/issues /tests/sentry/tasks/test_auto_ongoing_issues.py @getsentry/issues From 551588c3f815d8aae6520d2a303a018e5be1b8ad Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Tue, 14 Jan 2025 10:12:46 -0500 Subject: [PATCH 15/22] Fix tests --- .../endpoints/issues/test_organization_derive_code_mappings.py | 2 +- tests/sentry/integrations/github/tasks/test_pr_comment.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/sentry/api/endpoints/issues/test_organization_derive_code_mappings.py b/tests/sentry/api/endpoints/issues/test_organization_derive_code_mappings.py index 8c196ba148d53d..8dc4542101c198 100644 --- a/tests/sentry/api/endpoints/issues/test_organization_derive_code_mappings.py +++ b/tests/sentry/api/endpoints/issues/test_organization_derive_code_mappings.py @@ -102,7 +102,7 @@ def test_get_multiple_matches(self, mock_get_trees_for_org): }, ] with patch( - "sentry.issues.auto_source_code_config.CodeMappingTreesHelper.list_file_matches", + "sentry.issues.auto_source_code_config.code_mapping.CodeMappingTreesHelper.list_file_matches", return_value=expected_matches, ): response = self.client.get(self.url, data=config_data, format="json") diff --git a/tests/sentry/integrations/github/tasks/test_pr_comment.py b/tests/sentry/integrations/github/tasks/test_pr_comment.py index 4cbb901678c21b..54bcee6d5374d0 100644 --- a/tests/sentry/integrations/github/tasks/test_pr_comment.py +++ b/tests/sentry/integrations/github/tasks/test_pr_comment.py @@ -403,7 +403,7 @@ def test_format_comment(self): expected_comment = """## Suspect Issues This pull request was deployed and Sentry observed the following issues: -- ‼️ **TypeError** `sentry.tasks.auto_source_code_config.derive_co...` [View Issue](https://sentry.sentry.io/issues/?referrer=github-pr-bot) +- ‼️ **TypeError** `sentry.tasks.auto_source_code_config.derive_cod...` [View Issue](https://sentry.sentry.io/issues/?referrer=github-pr-bot) - ‼️ **KafkaException** `query_subscription_consumer_process_message` [View Issue](https://sentry.sentry.io/stats/?referrer=github-pr-bot) Did you find this useful? React with a 👍 or 👎""" From ebf3d8ae2aef3cc73f516d51c88853b0950f384b Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Tue, 14 Jan 2025 10:21:19 -0500 Subject: [PATCH 16/22] Rename back --- src/sentry/tasks/auto_source_code_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentry/tasks/auto_source_code_config.py b/src/sentry/tasks/auto_source_code_config.py index 5f8ac6c1e41594..55a4bd9685e163 100644 --- a/src/sentry/tasks/auto_source_code_config.py +++ b/src/sentry/tasks/auto_source_code_config.py @@ -84,7 +84,7 @@ def process_error(error: ApiError, extra: dict[str, str]) -> None: @instrumented_task( - name="sentry.tasks.auto_source_code_config.derive_code_mappings", + name="sentry.tasks.derive_code_mappings.derive_code_mappings", # XXX: To be renamed to auto_source_code_config queue="derive_code_mappings", # XXX: To be renamed to auto_source_code_config default_retry_delay=60 * 10, max_retries=3, From 8d0fc95cda337dbed29e9d1896a2c416337724dc Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Tue, 14 Jan 2025 14:12:17 -0500 Subject: [PATCH 17/22] Old and new task --- src/sentry/tasks/auto_source_code_config.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/sentry/tasks/auto_source_code_config.py b/src/sentry/tasks/auto_source_code_config.py index 55a4bd9685e163..7a21ee6df6cfc5 100644 --- a/src/sentry/tasks/auto_source_code_config.py +++ b/src/sentry/tasks/auto_source_code_config.py @@ -83,15 +83,30 @@ def process_error(error: ApiError, extra: dict[str, str]) -> None: ) +# XXX: To be deleted after new queue is live @instrumented_task( - name="sentry.tasks.derive_code_mappings.derive_code_mappings", # XXX: To be renamed to auto_source_code_config - queue="derive_code_mappings", # XXX: To be renamed to auto_source_code_config + name="sentry.tasks.derive_code_mappings.derive_code_mappings", + queue="derive_code_mappings", default_retry_delay=60 * 10, max_retries=3, ) def derive_code_mappings( project_id: int, data: NodeData, +) -> None: + derive_code_mappings_new(project_id, data) + + +# XXX: Temporary, use the new queue when live +@instrumented_task( + name="sentry.tasks.derive_code_mappings.derive_code_mappings", + queue="derive_code_mappings", + default_retry_delay=60 * 10, + max_retries=3, +) +def derive_code_mappings_new( + project_id: int, + data: NodeData, ) -> None: """ Derive code mappings for a project given data from a recent event. From e2d3b6ed5e385482c79aaab964ce4c35057f343a Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Tue, 14 Jan 2025 14:17:01 -0500 Subject: [PATCH 18/22] Remove SUPPORTED_LANGUAGES import --- src/sentry/tasks/auto_source_code_config.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/sentry/tasks/auto_source_code_config.py b/src/sentry/tasks/auto_source_code_config.py index 43fef24b3008c0..f804ff07224085 100644 --- a/src/sentry/tasks/auto_source_code_config.py +++ b/src/sentry/tasks/auto_source_code_config.py @@ -16,11 +16,7 @@ SCMIntegrationInteractionEvent, SCMIntegrationInteractionType, ) -from sentry.issues.auto_source_code_config.code_mapping import ( - SUPPORTED_LANGUAGES, - CodeMapping, - CodeMappingTreesHelper, -) +from sentry.issues.auto_source_code_config.code_mapping import CodeMapping, CodeMappingTreesHelper from sentry.locks import locks from sentry.models.organization import Organization from sentry.models.project import Project From 3afe2d7a5f9058dac97dd33f97f315a610feff83 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Tue, 14 Jan 2025 20:04:22 -0500 Subject: [PATCH 19/22] Fix name --- src/sentry/tasks/auto_source_code_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentry/tasks/auto_source_code_config.py b/src/sentry/tasks/auto_source_code_config.py index f804ff07224085..dacb53fa6e1c2e 100644 --- a/src/sentry/tasks/auto_source_code_config.py +++ b/src/sentry/tasks/auto_source_code_config.py @@ -94,7 +94,7 @@ def derive_code_mappings( # XXX: Temporary, use the new queue when live @instrumented_task( - name="sentry.tasks.derive_code_mappings.derive_code_mappings", + name="sentry.tasks.derive_code_mappings.derive_code_mappings_new", queue="derive_code_mappings", default_retry_delay=60 * 10, max_retries=3, From 338a1058ab6899a1170587014496f461e4df8a24 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Wed, 15 Jan 2025 13:10:46 -0500 Subject: [PATCH 20/22] Use new queue --- src/sentry/tasks/auto_source_code_config.py | 16 ++++++++-------- src/sentry/tasks/post_process.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/sentry/tasks/auto_source_code_config.py b/src/sentry/tasks/auto_source_code_config.py index dacb53fa6e1c2e..53e8b9c356c210 100644 --- a/src/sentry/tasks/auto_source_code_config.py +++ b/src/sentry/tasks/auto_source_code_config.py @@ -78,7 +78,7 @@ def process_error(error: ApiError, extra: dict[str, str]) -> None: ) -# XXX: To be deleted after new queue is live +# XXX: To be deleted after queue is empty @instrumented_task( name="sentry.tasks.derive_code_mappings.derive_code_mappings", queue="derive_code_mappings", @@ -89,24 +89,24 @@ def derive_code_mappings( project_id: int, data: NodeData, ) -> None: - derive_code_mappings_new(project_id, data) + auto_source_code_config(project_id, data) -# XXX: Temporary, use the new queue when live @instrumented_task( - name="sentry.tasks.derive_code_mappings.derive_code_mappings_new", - queue="derive_code_mappings", + name="sentry.tasks.auto_source_code_config", + queue="auto_source_code_config", default_retry_delay=60 * 10, max_retries=3, ) -def derive_code_mappings_new( +def auto_source_code_config( project_id: int, data: NodeData, ) -> None: """ - Derive code mappings for a project given data from a recent event. + Process errors for customers with source code management installed and calculate code mappings + among other things. - This task is queued at most once per hour per project, based on the ingested events. + This task is queued at most once per hour per project. """ project = Project.objects.get(id=project_id) org: Organization = Organization.objects.get(id=project.organization_id) diff --git a/src/sentry/tasks/post_process.py b/src/sentry/tasks/post_process.py index 44976a4e00708f..400c2da19b0025 100644 --- a/src/sentry/tasks/post_process.py +++ b/src/sentry/tasks/post_process.py @@ -995,7 +995,7 @@ def process_code_mappings(job: PostProcessJob) -> None: return from sentry.issues.auto_source_code_config.code_mapping import SUPPORTED_LANGUAGES - from sentry.tasks.auto_source_code_config import derive_code_mappings + from sentry.tasks.auto_source_code_config import auto_source_code_config try: event = job["event"] @@ -1017,7 +1017,7 @@ def process_code_mappings(job: PostProcessJob) -> None: else: return - derive_code_mappings.delay(project.id, event.data) + auto_source_code_config.delay(project.id, event.data) except Exception: logger.exception("derive_code_mappings: Failed to process code mappings") From 713332d0b60d165d15a1694df5c03f7d554d4ade Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Wed, 15 Jan 2025 15:15:42 -0500 Subject: [PATCH 21/22] Use flag to switch over --- src/sentry/features/temporary.py | 2 ++ src/sentry/tasks/post_process.py | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/sentry/features/temporary.py b/src/sentry/features/temporary.py index 909e6cf2cbf8b0..f48357f23a5684 100644 --- a/src/sentry/features/temporary.py +++ b/src/sentry/features/temporary.py @@ -34,6 +34,8 @@ def register_temporary_features(manager: FeatureManager): # Enables user registration. manager.add("auth:register", SystemFeature, FeatureHandlerStrategy.INTERNAL, default=True) + # Switch to new queue for auto source code config + manager.add("new-auto-source-code-config-queue", SystemFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False) # Enable creating organizations within sentry (if SENTRY_SINGLE_ORGANIZATION is not enabled). manager.add("organizations:create", SystemFeature, FeatureHandlerStrategy.INTERNAL, default=True) # Controls whether or not the relocation endpoints can be used. diff --git a/src/sentry/tasks/post_process.py b/src/sentry/tasks/post_process.py index 400c2da19b0025..0002acffedd9fa 100644 --- a/src/sentry/tasks/post_process.py +++ b/src/sentry/tasks/post_process.py @@ -995,7 +995,7 @@ def process_code_mappings(job: PostProcessJob) -> None: return from sentry.issues.auto_source_code_config.code_mapping import SUPPORTED_LANGUAGES - from sentry.tasks.auto_source_code_config import auto_source_code_config + from sentry.tasks.auto_source_code_config import auto_source_code_config, derive_code_mappings try: event = job["event"] @@ -1017,7 +1017,10 @@ def process_code_mappings(job: PostProcessJob) -> None: else: return - auto_source_code_config.delay(project.id, event.data) + if features.has("new-auto-source-code-config-queue"): + auto_source_code_config.delay(project.id, event.data) + else: + derive_code_mappings.delay(project.id, event.data) except Exception: logger.exception("derive_code_mappings: Failed to process code mappings") From a723dc5aa8795fbf400ac587644f7affc5708388 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G <44410+armenzg@users.noreply.github.com> Date: Wed, 15 Jan 2025 15:51:21 -0500 Subject: [PATCH 22/22] Add test --- src/sentry/features/temporary.py | 2 +- tests/sentry/tasks/test_post_process.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/sentry/features/temporary.py b/src/sentry/features/temporary.py index f48357f23a5684..53e4da3417f009 100644 --- a/src/sentry/features/temporary.py +++ b/src/sentry/features/temporary.py @@ -35,7 +35,7 @@ def register_temporary_features(manager: FeatureManager): # Enables user registration. manager.add("auth:register", SystemFeature, FeatureHandlerStrategy.INTERNAL, default=True) # Switch to new queue for auto source code config - manager.add("new-auto-source-code-config-queue", SystemFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False) + manager.add("new-auto-source-code-config-queue", SystemFeature, FeatureHandlerStrategy.FLAGPOLE) # Enable creating organizations within sentry (if SENTRY_SINGLE_ORGANIZATION is not enabled). manager.add("organizations:create", SystemFeature, FeatureHandlerStrategy.INTERNAL, default=True) # Controls whether or not the relocation endpoints can be used. diff --git a/tests/sentry/tasks/test_post_process.py b/tests/sentry/tasks/test_post_process.py index 83e7d39ad4608a..b343c62d9aa25a 100644 --- a/tests/sentry/tasks/test_post_process.py +++ b/tests/sentry/tasks/test_post_process.py @@ -375,6 +375,28 @@ def test_skipping_an_issue_doesnt_mark_it_processed(self, mock_derive_code_mappi self._call_post_process_group(charlie_event2) assert mock_derive_code_mappings.delay.call_count == 2 + # XXX: Delete this test once we've migrated + @patch("sentry.tasks.auto_source_code_config.auto_source_code_config") + @patch("sentry.tasks.auto_source_code_config.derive_code_mappings") + def test_new_queue(self, mock_derive_code_mappings, mock_auto_source_code_config): + event = self._create_event(data={}, project_id=self.project.id) + + with self.feature({"new-auto-source-code-config-queue": False}): + self._call_post_process_group(event) + assert mock_derive_code_mappings.delay.call_count == 1 + assert mock_auto_source_code_config.delay.call_count == 0 + + # XXX: Delete this test once we've migrated + @patch("sentry.tasks.auto_source_code_config.auto_source_code_config") + @patch("sentry.tasks.auto_source_code_config.derive_code_mappings") + def test_old_queue(self, mock_derive_code_mappings, mock_auto_source_code_config): + event = self._create_event(data={}, project_id=self.project.id) + + with self.feature({"new-auto-source-code-config-queue": True}): + self._call_post_process_group(event) + assert mock_derive_code_mappings.delay.call_count == 0 + assert mock_auto_source_code_config.delay.call_count == 1 + class RuleProcessorTestMixin(BasePostProgressGroupMixin): @patch("sentry.rules.processing.processor.RuleProcessor")