Add contextual comments to CodeTF when adding dependencies

integration_tests/base_test.py

@@ -101,7 +101,7 @@ def _assert_results_fields(self, results, output_path):
 
         assert len(change["changes"]) == self.num_changes
         line_change = change["changes"][0]
-        assert line_change["lineNumber"] == self.expected_line_change
+        assert line_change["lineNumber"] == str(self.expected_line_change)
         assert line_change["description"] == self.change_description
         assert line_change["packageActions"] == []
         assert line_change["properties"] == {}

integration_tests/test_process_sandbox.py

@@ -26,4 +26,4 @@ class TestProcessSandbox(BaseIntegrationTest):
 
     requirements_path = "tests/samples/requirements.txt"
     original_requirements = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\n"
-    expected_new_reqs = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\nsecurity==1.0.1"
+    expected_new_reqs = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\nsecurity~=1.2.0"

integration_tests/test_url_sandbox.py

@@ -23,4 +23,4 @@ class TestUrlSandbox(BaseIntegrationTest):
 
     requirements_path = "tests/samples/requirements.txt"
     original_requirements = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\n"
-    expected_new_reqs = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\nsecurity==1.0.1"
+    expected_new_reqs = "# file used to test dependency management\nrequests==2.31.0\nblack==23.7.*\nmypy~=1.4\npylint>1\nsecurity~=1.2.0"

src/codemodder/change.py

@@ -1,19 +1,46 @@
 from dataclasses import dataclass, field
+from enum import Enum
+
+
+class Action(Enum):
+    ADD = "add"
+    REMOVE = "remove"
+
+
+class Result(Enum):
+    COMPLETED = "completed"
+    FAILED = "failed"
+    SKIPPED = "skipped"
+
+
+@dataclass
+class PackageAction:
+    action: Action
+    result: Result
+    package: str
+
+    def to_json(self):
+        return {
+            "action": self.action.value,
+            "result": self.result.value,
+            "package": self.package,
+        }
 
 
 @dataclass
 class Change:
-    lineNumber: str
+    lineNumber: int
     description: str
     properties: dict = field(default_factory=dict)
-    packageActions: list = field(default_factory=list)
+    packageActions: list[PackageAction] = field(default_factory=list)
 
     def to_json(self):
         return {
-            "lineNumber": self.lineNumber,
+            # Not sure why this is a string but it's in the spec
+            "lineNumber": str(self.lineNumber),
             "description": self.description,
             "properties": self.properties,
-            "packageActions": self.packageActions,
+            "packageActions": [pa.to_json() for pa in self.packageActions],
         }
 
 
@@ -26,4 +53,8 @@
     changes: list[Change]
 
     def to_json(self):
-        return {"path": self.path, "diff": self.diff, "changes": self.changes}
+        return {
+            "path": self.path,
+            "diff": self.diff,
+            "changes": [x.to_json() for x in self.changes],
+        }

src/codemodder/codemods/api/__init__.py

@@ -96,7 +96,7 @@ def __init__(
     def report_change(self, original_node):
         line_number = self.lineno_for_node(original_node)
         self.file_context.codemod_changes.append(
-            Change(str(line_number), self.CHANGE_DESCRIPTION).to_json()
+            Change(line_number, self.CHANGE_DESCRIPTION)
         )
 
 

src/codemodder/codemods/base_codemod.py

@@ -6,6 +6,7 @@
 
 from codemodder.change import Change
 from codemodder.context import CodemodExecutionContext
+from codemodder.dependency import Dependency
 from codemodder.file_context import FileContext
 from codemodder.semgrep import run as semgrep_run
 
@@ -43,7 +44,8 @@ class BaseCodemod:
     # Implementation borrowed from https://stackoverflow.com/a/45250114
     METADATA: ClassVar[CodemodMetadata] = NotImplemented
     SUMMARY: ClassVar[str] = NotImplemented
-    is_semgrep = False
+    is_semgrep: bool = False
+    adds_dependency: bool = False
 
     execution_context: CodemodExecutionContext
     file_context: FileContext
@@ -85,7 +87,7 @@ def add_change_from_position(self, position: CodeRange, description):
             Change(
                 lineNumber=position.start.line,
                 description=description,
-            ).to_json()
+            )
         )
 
     def lineno_for_node(self, node):
@@ -99,7 +101,7 @@ def line_exclude(self):
     def line_include(self):
         return self.file_context.line_include
 
-    def add_dependency(self, dependency: str):
+    def add_dependency(self, dependency: Dependency):
         self.file_context.add_dependency(dependency)
 
 

src/codemodder/codemods/imported_call_modifier.py

@@ -35,7 +35,7 @@ def __init__(
         self.line_include = file_context.line_include
         self.matching_functions: FunctionMatchType = matching_functions
         self.change_description = change_description
-        self.changes_in_file: list[Mapping] = []
+        self.changes_in_file: list[Change] = []
 
     def updated_args(self, original_args: Sequence[cst.Arg]):
         return original_args
@@ -71,7 +71,7 @@ def leave_Call(self, original_node: cst.Call, updated_node: cst.Call):
                 and true_name in self.matching_functions
             ):
                 self.changes_in_file.append(
-                    Change(str(line_number), self.change_description).to_json()
+                    Change(line_number, self.change_description)
                 )
 
                 new_args = self.updated_args(updated_node.args)

src/codemodder/context.py

@@ -4,16 +4,30 @@
 from textwrap import indent
 
 from codemodder.change import ChangeSet
+from codemodder.dependency import Dependency
 from codemodder.dependency_manager import DependencyManager
 from codemodder.executor import CodemodExecutorWrapper
 from codemodder.logging import logger, log_list
 from codemodder.registry import CodemodRegistry
 
 
+DEPENDENCY_NOTIFICATION = """```
+💡 This codemod adds a dependency to your project. \
+Currently we add the dependency to a file named `requirements.txt` if it \
+exists in your project.
+
+There are a number of other places where Python project dependencies can be \
+expressed, including `setup.py`, `pyproject.toml`, and `setup.cfg`. We are \
+working on adding support for these files, but for now you may need to update \
+these files manually before accepting this change.
+```
+"""
+
+
 class CodemodExecutionContext:  # pylint: disable=too-many-instance-attributes
     _results_by_codemod: dict[str, list[ChangeSet]] = {}
     _failures_by_codemod: dict[str, list[Path]] = {}
-    dependencies: dict[str, set[str]] = {}
+    dependencies: dict[str, set[Dependency]] = {}
     directory: Path
     dry_run: bool = False
     verbose: bool = False
@@ -40,7 +54,7 @@ def add_result(self, codemod_name, change_set):
     def add_failure(self, codemod_name, file_path):
         self._failures_by_codemod.setdefault(codemod_name, []).append(file_path)
 
-    def add_dependencies(self, codemod_id: str, dependencies: set[str]):
+    def add_dependencies(self, codemod_id: str, dependencies: set[Dependency]):
         self.dependencies.setdefault(codemod_id, set()).update(dependencies)
 
     def get_results(self, codemod_name):
@@ -80,13 +94,20 @@ def process_dependencies(self, codemod_id: str):
         if (changeset := dm.write(self.dry_run)) is not None:
             self.add_result(codemod_id, changeset)
 
+    def add_description(self, codemod: CodemodExecutorWrapper):
+        description = codemod.description
+        if codemod.adds_dependency:
+            description = f"{description}\n\n{DEPENDENCY_NOTIFICATION}"
+
+        return description
+
     def compile_results(self, codemods: list[CodemodExecutorWrapper]):
         results = []
         for codemod in codemods:
             data = {
                 "codemod": codemod.id,
                 "summary": codemod.summary,
-                "description": codemod.description,
+                "description": self.add_description(codemod),
                 "references": codemod.references,
                 "properties": {},
                 "failedFiles": [str(file) for file in self.get_failures(codemod.id)],

src/codemodder/dependency.py

@@ -0,0 +1,63 @@
+from dataclasses import dataclass
+
+from packaging.requirements import Requirement
+
+
+@dataclass
+class License:
+    name: str
+    url: str
+
+
+@dataclass
+class Dependency:
+    requirement: Requirement
+    description: str
+    _license: License
+    oss_link: str
+    package_link: str
+
+    @property
+    def name(self) -> str:
+        return self.requirement.name
+
+    @property
+    def version(self) -> str:
+        return self.requirement.specifier.__str__().strip()
+
+    def build_description(self) -> str:
+        return f"""{self.description}
+
+License: [{self._license.name}]({self._license.url}) ✅ \
+[Open Source]({self.oss_link}) ✅ \
+[More facts]({self.package_link})
+"""
+
+    def __hash__(self):
+        return hash(self.requirement)
+
+
+DefusedXML = Dependency(
+    Requirement("defusedxml~=0.7.1"),
+    description="""\
+This package is [recommended by the Python community](https://docs.python.org/3/library/xml.html#the-defusedxml-package) \
+to protect against XML vulnerabilities.\
+""",
+    _license=License(
+        "PSF-2.0",
+        "https://opensource.org/license/python-2-0/",
+    ),
+    oss_link="https://github.com/tiran/defusedxml",
+    package_link="https://pypi.org/project/defusedxml/",
+)
+
+Security = Dependency(
+    Requirement("security~=1.2.0"),
+    description="""This library holds security tools for protecting Python API calls.""",
+    _license=License(
+        "MIT",
+        "https://opensource.org/license/MIT/",
+    ),
+    oss_link="https://github.com/pixee/python-security",
+    package_link="https://pypi.org/project/security/",
+)

src/codemodder/dependency_manager.py

@@ -5,13 +5,14 @@
 import difflib
 from packaging.requirements import Requirement
 
-from codemodder.change import ChangeSet
+from codemodder.change import Action, Change, ChangeSet, PackageAction, Result
+from codemodder.dependency import Dependency
 
 
 class DependencyManager:
     parent_directory: Path
     _lines: list[str]
-    _new_requirements: list[str]
+    _new_requirements: list[Dependency]
 
     def __init__(self, parent_directory: Path):
         """One-time class initialization."""
@@ -20,13 +21,16 @@ def __init__(self, parent_directory: Path):
         self._lines = []
         self._new_requirements = []
 
-    def add(self, dependencies: list[str]):
+    @property
+    def new_requirements(self) -> list[str]:
+        return [str(x.requirement) for x in self._new_requirements]
+
+    def add(self, dependencies: list[Dependency]):
         """add any number of dependencies to the end of list of dependencies."""
-        for dep_str in dependencies:
-            dep = Requirement(dep_str)
+        for dep in dependencies:
             if dep not in self.dependencies:
-                self.dependencies.update({dep: None})
-                self._new_requirements.append(str(dep))
+                self.dependencies.update({dep.requirement: None})
+                self._new_requirements.append(dep)
 
     def write(self, dry_run: bool = False) -> Optional[ChangeSet]:
         """
@@ -35,19 +39,29 @@ def write(self, dry_run: bool = False) -> Optional[ChangeSet]:
         if not (self.dependency_file and self._new_requirements):
             return None
 
-        updated = self._lines + self._new_requirements + ["\n"]
+        updated = self._lines + self.new_requirements + ["\n"]
 
         diff = "".join(difflib.unified_diff(self._lines, updated))
-        # TODO: add a change entry for each new requirement
-        # TODO: make sure to set the contextual_description=True in the properties bag
+
+        changes = [
+            Change(
+                lineNumber=len(self._lines) + i + 1,
+                description=dep.build_description(),
+                properties={"contextual_description": True},
+                packageActions=[
+                    PackageAction(Action.ADD, Result.COMPLETED, str(dep.requirement))
+                ],
+            )
+            for i, dep in enumerate(self._new_requirements)
+        ]
 
         if not dry_run:
             with open(self.dependency_file, "w", encoding="utf-8") as f:
                 f.writelines(self._lines)
-                f.writelines(self._new_requirements)
+                f.writelines(self.new_requirements)
 
         self.dependency_file_changed = True
-        return ChangeSet(str(self.dependency_file), diff, changes=[])
+        return ChangeSet(str(self.dependency_file), diff, changes=changes)
 
     @property
     def found_dependency_file(self) -> bool:

src/codemodder/file_context.py

@@ -2,6 +2,9 @@
 from pathlib import Path
 from typing import Dict, List
 
+from codemodder.change import Change
+from codemodder.dependency import Dependency
+
 
 @dataclass
 class FileContext:
@@ -10,17 +13,11 @@ class FileContext:
     """
 
     file_path: Path
-    line_exclude: List[int]
-    line_include: List[int]
-    results_by_id: Dict
-    dependencies: set[str] = field(default_factory=set)
-
-    def __post_init__(self):
-        if self.line_include is None:
-            self.line_include = []
-        if self.line_exclude is None:
-            self.line_exclude = []
-        self.codemod_changes = []
+    line_exclude: List[int] = field(default_factory=list)
+    line_include: List[int] = field(default_factory=list)
+    results_by_id: Dict = field(default_factory=dict)
+    dependencies: set[Dependency] = field(default_factory=set)
+    codemod_changes: List[Change] = field(default_factory=list)
 
-    def add_dependency(self, dependency: str):
+    def add_dependency(self, dependency: Dependency):
         self.dependencies.add(dependency)

src/core_codemods/django_debug_flag_on.py

@@ -74,7 +74,7 @@ def leave_Assign(
         ) and self.filter_by_path_includes_or_excludes(pos_to_match):
             line_number = pos_to_match.start.line
             self.changes_in_file.append(
-                Change(str(line_number), DjangoDebugFlagOn.CHANGE_DESCRIPTION).to_json()
+                Change(line_number, DjangoDebugFlagOn.CHANGE_DESCRIPTION)
             )
             return updated_node.with_changes(value=cst.Name("False"))
         return updated_node