ref(derived_code_mappings): Rename to Automatic Source Code Configuration

.github/CODEOWNERS

@@ -519,7 +519,6 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py     @ge
 
 ## Issues
 **/issues/**                                                @getsentry/issues
-*code_mappings*.py                                          @getsentry/issues
 /src/sentry/api/helpers/actionable_items_helper.py          @getsentry/issues
 /src/sentry/api/helpers/events.py                           @getsentry/issues
 /src/sentry/api/helpers/group_index/                        @getsentry/issues
@@ -539,6 +538,7 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py     @ge
 /src/sentry/tasks/auto_ongoing_issues.py                    @getsentry/issues
 /src/sentry/tasks/auto_remove_inbox.py                      @getsentry/issues
 /src/sentry/tasks/auto_resolve_issues.py                    @getsentry/issues
+/src/sentry/tasks/auto_source_code_config.py                @getsentry/issues
 /src/sentry/tasks/check_new_issue_threshold_met.py          @getsentry/issues
 /src/sentry/tasks/clear_expired_resolutions.py              @getsentry/issues
 /src/sentry/tasks/clear_expired_rulesnoozes.py              @getsentry/issues
@@ -547,7 +547,6 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py     @ge
 /src/sentry/tasks/commit_context.py                         @getsentry/issues
 /src/sentry/tasks/commits.py                                @getsentry/issues
 /src/sentry/tasks/delete_seer_grouping_records.py           @getsentry/issues
-/src/sentry/tasks/derive_code_mappings.py                   @getsentry/issues
 /src/sentry/tasks/embeddings_grouping/                      @getsentry/issues
 /src/sentry/tasks/groupowner.py                             @getsentry/issues
 /src/sentry/tasks/merge.py                                  @getsentry/issues
@@ -574,6 +573,7 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py     @ge
 /tests/sentry/tasks/test_auto_ongoing_issues.py             @getsentry/issues
 /tests/sentry/tasks/test_auto_remove_inbox.py               @getsentry/issues
 /tests/sentry/tasks/test_auto_resolve_issues.py             @getsentry/issues
+/tests/sentry/tasks/test_auto_source_code_config.py         @getsentry/issues
 /tests/sentry/tasks/test_backfill_seer_grouping_records.py  @getsentry/issues
 /tests/sentry/tasks/test_check_new_issue_threshold_met.py   @getsentry/issues
 /tests/sentry/tasks/test_clear_expired_resolutions.py       @getsentry/issues
@@ -583,7 +583,6 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py     @ge
 /tests/sentry/tasks/test_commit_context.py                  @getsentry/issues
 /tests/sentry/tasks/test_commits.py                         @getsentry/issues
 /tests/sentry/tasks/test_delete_seer_grouping_records.py    @getsentry/issues
-/tests/sentry/tasks/test_derive_code_mappings.py            @getsentry/issues
 /tests/sentry/tasks/test_groupowner.py                      @getsentry/issues
 /tests/sentry/tasks/test_merge.py                           @getsentry/issues
 /tests/sentry/tasks/test_post_process.py                    @getsentry/issues

pyproject.toml

@@ -402,6 +402,7 @@ module = [
     "sentry.issues",
     "sentry.issues.analytics",
     "sentry.issues.apps",
+    "sentry.issues.auto_source_code_config.*",
     "sentry.issues.constants",
     "sentry.issues.endpoints",
     "sentry.issues.endpoints.actionable_items",
@@ -413,6 +414,7 @@ module = [
     "sentry.issues.endpoints.group_similar_issues_embeddings",
     "sentry.issues.endpoints.group_tombstone",
     "sentry.issues.endpoints.group_tombstone_details",
+    "sentry.issues.endpoints.organization_derive_code_mappings",
     "sentry.issues.endpoints.organization_eventid",
     "sentry.issues.endpoints.organization_group_index",
     "sentry.issues.endpoints.organization_group_index_stats",

src/sentry/api/endpoints/group_ai_autofix.py

@@ -17,7 +17,7 @@
 from sentry.api.bases.group import GroupEndpoint
 from sentry.api.serializers import EventSerializer, serialize
 from sentry.autofix.utils import get_autofix_repos_from_project_code_mappings, get_autofix_state
-from sentry.integrations.utils.code_mapping import get_sorted_code_mapping_configs
+from sentry.issues.auto_source_code_config.code_mapping import get_sorted_code_mapping_configs
 from sentry.models.group import Group
 from sentry.seer.signed_seer_api import get_seer_salted_url, sign_with_seer_secret
 from sentry.tasks.autofix import check_autofix_status

src/sentry/api/endpoints/group_autofix_setup_check.py

@@ -14,7 +14,7 @@
 from sentry.autofix.utils import get_autofix_repos_from_project_code_mappings
 from sentry.constants import ObjectStatus
 from sentry.integrations.services.integration import integration_service
-from sentry.integrations.utils.code_mapping import get_sorted_code_mapping_configs
+from sentry.issues.auto_source_code_config.code_mapping import get_sorted_code_mapping_configs
 from sentry.models.group import Group
 from sentry.models.organization import Organization
 from sentry.models.project import Project

src/sentry/api/endpoints/project_repo_path_parsing.py

@@ -12,7 +12,7 @@
 from sentry.integrations.base import IntegrationFeatures
 from sentry.integrations.manager import default_manager as integrations
 from sentry.integrations.services.integration import RpcIntegration, integration_service
-from sentry.integrations.utils.code_mapping import find_roots
+from sentry.issues.auto_source_code_config.code_mapping import find_roots
 from sentry.models.repository import Repository
 
 

src/sentry/api/endpoints/project_stacktrace_coverage.py

@@ -9,9 +9,9 @@
 from sentry.api.base import region_silo_endpoint
 from sentry.api.bases.project import ProjectEndpoint
 from sentry.api.serializers import serialize
-from sentry.integrations.utils.code_mapping import get_sorted_code_mapping_configs
 from sentry.integrations.utils.codecov import codecov_enabled, fetch_codecov_data
 from sentry.integrations.utils.stacktrace_link import get_stacktrace_config
+from sentry.issues.auto_source_code_config.code_mapping import get_sorted_code_mapping_configs
 from sentry.issues.endpoints.project_stacktrace_link import generate_context
 from sentry.models.project import Project
 from sentry.utils import metrics

src/sentry/api/urls.py

@@ -177,6 +177,7 @@
     GroupSimilarIssuesEndpoint,
     GroupTombstoneDetailsEndpoint,
     GroupTombstoneEndpoint,
+    OrganizationDeriveCodeMappingsEndpoint,
     OrganizationGroupIndexEndpoint,
     OrganizationGroupIndexStatsEndpoint,
     OrganizationGroupSearchViewsEndpoint,
@@ -448,7 +449,6 @@
     OrganizationDashboardWidgetDetailsEndpoint,
 )
 from .endpoints.organization_dashboards import OrganizationDashboardsEndpoint
-from .endpoints.organization_derive_code_mappings import OrganizationDeriveCodeMappingsEndpoint
 from .endpoints.organization_details import OrganizationDetailsEndpoint
 from .endpoints.organization_environments import OrganizationEnvironmentsEndpoint
 from .endpoints.organization_event_details import OrganizationEventDetailsEndpoint

src/sentry/autofix/utils.py

@@ -7,7 +7,7 @@
 from django.conf import settings
 from pydantic import BaseModel
 
-from sentry.integrations.utils.code_mapping import get_sorted_code_mapping_configs
+from sentry.issues.auto_source_code_config.code_mapping import get_sorted_code_mapping_configs
 from sentry.models.project import Project
 from sentry.models.repository import Repository
 from sentry.seer.signed_seer_api import get_seer_salted_url, sign_with_seer_secret

src/sentry/conf/server.py

@@ -819,7 +819,7 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str:
     "sentry.dynamic_sampling.tasks.sliding_window_org",
     "sentry.dynamic_sampling.tasks.utils",
     "sentry.dynamic_sampling.tasks.custom_rule_notifications",
-    "sentry.tasks.derive_code_mappings",
+    "sentry.tasks.auto_source_code_config",
     "sentry.ingest.transaction_clusterer.tasks",
     "sentry.tasks.auto_enable_codecov",
     "sentry.tasks.weekly_escalating_forecast",
@@ -982,7 +982,9 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str:
     Queue("replays.delete_replay", routing_key="replays.delete_replay"),
     Queue("counters-0", routing_key="counters-0"),
     Queue("triggers-0", routing_key="triggers-0"),
+    # XXX: Temporarilty keep in place until we have migrated to the new queue
     Queue("derive_code_mappings", routing_key="derive_code_mappings"),
+    Queue("auto_source_code_config", routing_key="auto_source_code_config"),
     Queue("transactions.name_clusterer", routing_key="transactions.name_clusterer"),
     Queue("auto_enable_codecov", routing_key="auto_enable_codecov"),
     Queue("weekly_escalating_forecast", routing_key="weekly_escalating_forecast"),

src/sentry/features/temporary.py

@@ -34,6 +34,8 @@ def register_temporary_features(manager: FeatureManager):
 
     # Enables user registration.
     manager.add("auth:register", SystemFeature, FeatureHandlerStrategy.INTERNAL, default=True)
+    # Switch to new queue for auto source code config
+    manager.add("new-auto-source-code-config-queue", SystemFeature, FeatureHandlerStrategy.FLAGPOLE)
     # Enable creating organizations within sentry (if SENTRY_SINGLE_ORGANIZATION is not enabled).
     manager.add("organizations:create", SystemFeature, FeatureHandlerStrategy.INTERNAL, default=True)
     # Controls whether or not the relocation endpoints can be used.

src/sentry/integrations/github/client.py

@@ -26,7 +26,7 @@
 )
 from sentry.integrations.source_code_management.repository import RepositoryClient
 from sentry.integrations.types import EXTERNAL_PROVIDERS, ExternalProviders
-from sentry.integrations.utils.code_mapping import (
+from sentry.issues.auto_source_code_config.code_mapping import (
     MAX_CONNECTION_ERRORS,
     Repo,
     RepoTree,

src/sentry/integrations/github/integration.py

@@ -32,11 +32,11 @@
 from sentry.integrations.source_code_management.commit_context import CommitContextIntegration
 from sentry.integrations.source_code_management.repository import RepositoryIntegration
 from sentry.integrations.tasks.migrate_repo import migrate_repo
-from sentry.integrations.utils.code_mapping import RepoTree
 from sentry.integrations.utils.metrics import (
     IntegrationPipelineViewEvent,
     IntegrationPipelineViewType,
 )
+from sentry.issues.auto_source_code_config.code_mapping import RepoTree
 from sentry.models.repository import Repository
 from sentry.organizations.absolute_url import generate_organization_url
 from sentry.organizations.services.organization import RpcOrganizationSummary, organization_service

src/sentry/integrations/utils/commit_context.py

@@ -18,7 +18,9 @@
     FileBlameInfo,
     SourceLineInfo,
 )
-from sentry.integrations.utils.code_mapping import convert_stacktrace_frame_path_to_source_path
+from sentry.issues.auto_source_code_config.code_mapping import (
+    convert_stacktrace_frame_path_to_source_path,
+)
 from sentry.models.commit import Commit
 from sentry.models.commitauthor import CommitAuthor
 from sentry.shared_integrations.exceptions import ApiError

src/sentry/integrations/utils/stacktrace_link.py

@@ -7,7 +7,9 @@
 from sentry.integrations.models.repository_project_path_config import RepositoryProjectPathConfig
 from sentry.integrations.services.integration import integration_service
 from sentry.integrations.source_code_management.repository import RepositoryIntegration
-from sentry.integrations.utils.code_mapping import convert_stacktrace_frame_path_to_source_path
+from sentry.issues.auto_source_code_config.code_mapping import (
+    convert_stacktrace_frame_path_to_source_path,
+)
 from sentry.models.repository import Repository
 from sentry.shared_integrations.exceptions import ApiError
 from sentry.utils.event_frames import EventFrame

src/sentry/integrations/utils/code_mapping.py → src/sentry/issues/auto_source_code_config/code_mapping.py

@@ -13,6 +13,8 @@
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
+SUPPORTED_LANGUAGES = ["javascript", "python", "node", "ruby", "php", "go", "csharp"]
+
 SLASH = "/"
 BACKSLASH = "\\"  # This is the Python representation of a single backslash
 
@@ -109,7 +111,9 @@ def __init__(self, frame_file_path: str) -> None:
     def __repr__(self) -> str:
         return f"FrameFilename: {self.full_path}"
 
-    def __eq__(self, other) -> bool:
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, FrameFilename):
+            return False
         return self.full_path == other.full_path
 
 

src/sentry/issues/endpoints/__init__.py

@@ -10,6 +10,7 @@
 from .group_similar_issues_embeddings import GroupSimilarIssuesEmbeddingsEndpoint
 from .group_tombstone import GroupTombstoneEndpoint
 from .group_tombstone_details import GroupTombstoneDetailsEndpoint
+from .organization_derive_code_mappings import OrganizationDeriveCodeMappingsEndpoint
 from .organization_eventid import EventIdLookupEndpoint
 from .organization_group_index import OrganizationGroupIndexEndpoint
 from .organization_group_index_stats import OrganizationGroupIndexStatsEndpoint
@@ -43,6 +44,7 @@
     "GroupSimilarIssuesEndpoint",
     "GroupTombstoneDetailsEndpoint",
     "GroupTombstoneEndpoint",
+    "OrganizationDeriveCodeMappingsEndpoint",
     "OrganizationGroupIndexEndpoint",
     "OrganizationGroupIndexStatsEndpoint",
     "OrganizationGroupSearchViewsEndpoint",

src/sentry/api/endpoints/organization_derive_code_mappings.py → src/sentry/issues/endpoints/organization_derive_code_mappings.py

@@ -11,7 +11,7 @@
 )
 from sentry.api.serializers import serialize
 from sentry.integrations.github.integration import GitHubIntegration
-from sentry.integrations.utils.code_mapping import (
+from sentry.issues.auto_source_code_config.code_mapping import (
     CodeMapping,
     CodeMappingTreesHelper,
     FrameFilename,
@@ -20,11 +20,16 @@
 )
 from sentry.models.organization import Organization
 from sentry.models.project import Project
-from sentry.tasks.derive_code_mappings import get_installation
+from sentry.tasks.auto_source_code_config import get_installation
 
 
 @region_silo_endpoint
 class OrganizationDeriveCodeMappingsEndpoint(OrganizationEndpoint):
+    """
+    In the UI, we have a prompt to derive code mappings from the stacktrace filename.
+    This endpoint is used to get the possible code mappings for it.
+    """
+
     owner = ApiOwner.ISSUES
     publish_status = {
         "GET": ApiPublishStatus.UNKNOWN,
@@ -42,7 +47,8 @@ def get(self, request: Request, organization: Organization) -> Response:
         :auth: required
         """
         stacktrace_filename = request.GET.get("stacktraceFilename")
-        installation, _ = get_installation(organization)  # only returns GitHub integrations
+        # It only returns the first GitHub integration
+        installation, _ = get_installation(organization)
         if not installation:
             return self.respond(
                 {"text": "Could not find this integration installed on your organization"},

src/sentry/issues/endpoints/project_stacktrace_link.py

@@ -17,8 +17,8 @@
 from sentry.integrations.api.serializers.models.integration import IntegrationSerializer
 from sentry.integrations.base import IntegrationFeatures
 from sentry.integrations.services.integration import integration_service
-from sentry.integrations.utils.code_mapping import get_sorted_code_mapping_configs
 from sentry.integrations.utils.stacktrace_link import StacktraceLinkOutcome, get_stacktrace_config
+from sentry.issues.auto_source_code_config.code_mapping import get_sorted_code_mapping_configs
 from sentry.models.project import Project
 
 logger = logging.getLogger(__name__)

src/sentry/tasks/derive_code_mappings.py → src/sentry/tasks/auto_source_code_config.py

@@ -17,7 +17,7 @@
     SCMIntegrationInteractionEvent,
     SCMIntegrationInteractionType,
 )
-from sentry.integrations.utils.code_mapping import CodeMapping, CodeMappingTreesHelper
+from sentry.issues.auto_source_code_config.code_mapping import CodeMapping, CodeMappingTreesHelper
 from sentry.locks import locks
 from sentry.models.organization import Organization
 from sentry.models.project import Project
@@ -27,8 +27,6 @@
 from sentry.utils.locking import UnableToAcquireLock
 from sentry.utils.safe import get_path
 
-SUPPORTED_LANGUAGES = ["javascript", "python", "node", "ruby", "php", "go", "csharp"]
-
 logger = logging.getLogger(__name__)
 
 if TYPE_CHECKING:
@@ -81,6 +79,7 @@ def process_error(error: ApiError, extra: dict[str, str]) -> None:
     )
 
 
+# XXX: To be deleted after queue is empty
 @instrumented_task(
     name="sentry.tasks.derive_code_mappings.derive_code_mappings",
     queue="derive_code_mappings",
@@ -91,11 +90,26 @@ def derive_code_mappings(
     project_id: int,
     data: NodeData | None = None,  # We will deprecate this
     event_id: str | None = None,
+) -> None:
+    auto_source_code_config(project_id, data, event_id)
+
+
+@instrumented_task(
+    name="sentry.tasks.auto_source_code_config",
+    queue="auto_source_code_config",
+    default_retry_delay=60 * 10,
+    max_retries=3,
+)
+def auto_source_code_config(
+    project_id: int,
+    data: NodeData | None = None,  # We will deprecate this
+    event_id: str | None = None,
 ) -> None:
     """
-    Derive code mappings for a project given data from a recent event.
+    Process errors for customers with source code management installed and calculate code mappings
+    among other things.
 
-    This task is queued at most once per hour per project, based on the ingested events.
+    This task is queued at most once per hour per project.
     """
     project = Project.objects.get(id=project_id)
     org: Organization = Organization.objects.get(id=project.organization_id)

src/sentry/tasks/commit_context.py

@@ -13,11 +13,11 @@
 from sentry import analytics
 from sentry.api.serializers.models.release import get_users_for_authors
 from sentry.integrations.source_code_management.commit_context import CommitContextIntegration
-from sentry.integrations.utils.code_mapping import get_sorted_code_mapping_configs
 from sentry.integrations.utils.commit_context import (
     find_commit_context_for_event_all_frames,
     get_or_create_commit_from_blame,
 )
+from sentry.issues.auto_source_code_config.code_mapping import get_sorted_code_mapping_configs
 from sentry.locks import locks
 from sentry.models.commit import Commit
 from sentry.models.commitauthor import CommitAuthor

src/sentry/tasks/post_process.py

@@ -994,7 +994,8 @@ def process_code_mappings(job: PostProcessJob) -> None:
     if job["is_reprocessed"]:
         return
 
-    from sentry.tasks.derive_code_mappings import SUPPORTED_LANGUAGES, derive_code_mappings
+    from sentry.issues.auto_source_code_config.code_mapping import SUPPORTED_LANGUAGES
+    from sentry.tasks.auto_source_code_config import auto_source_code_config, derive_code_mappings
 
     try:
         event = job["event"]
@@ -1016,8 +1017,10 @@ def process_code_mappings(job: PostProcessJob) -> None:
         else:
             return
 
-        # XXX: We will stop calling data after we deploy this change
-        derive_code_mappings.delay(project.id, data=event.data, event_id=event.event_id)
+        if features.has("new-auto-source-code-config-queue"):
+            auto_source_code_config.delay(project.id, data=event.data, event_id=event.event_id)
+        else:
+            derive_code_mappings.delay(project.id, data=event.data, event_id=event.event_id)
 
     except Exception:
         logger.exception("derive_code_mappings: Failed to process code mappings")

src/sentry/utils/sdk.py

@@ -49,6 +49,7 @@
 # Tasks not included here are sampled with `SENTRY_BACKEND_APM_SAMPLING`.
 # If a parent task schedules other tasks, rates propagate to the children.
 SAMPLED_TASKS = {
+    "sentry.tasks.auto_source_code_config.derive_code_mappings": settings.SAMPLED_DEFAULT_RATE,
     "sentry.tasks.send_ping": settings.SAMPLED_DEFAULT_RATE,
     "sentry.tasks.store.process_event": settings.SENTRY_PROCESS_EVENT_APM_SAMPLING,
     "sentry.tasks.store.process_event_from_reprocessing": settings.SENTRY_PROCESS_EVENT_APM_SAMPLING,
@@ -71,8 +72,6 @@
     "sentry.tasks.summaries.weekly_reports.prepare_organization_report": 0.1
     * settings.SENTRY_BACKEND_APM_SAMPLING,
     "sentry.profiles.task.process_profile": 0.1 * settings.SENTRY_BACKEND_APM_SAMPLING,
-    "sentry.tasks.derive_code_mappings.process_organizations": settings.SAMPLED_DEFAULT_RATE,
-    "sentry.tasks.derive_code_mappings.derive_code_mappings": settings.SAMPLED_DEFAULT_RATE,
     "sentry.monitors.tasks.clock_pulse": 1.0,
     "sentry.tasks.auto_enable_codecov": settings.SAMPLED_DEFAULT_RATE,
     "sentry.dynamic_sampling.tasks.boost_low_volume_projects": 1.0,