tests/repo: generate metadata file fixtures

.gitattributes

@@ -0,0 +1,2 @@
+# Do not mess with line endings in metadata files or the hash will be wrong.
+*.metadata binary

.pre-commit-config.yaml

@@ -6,8 +6,9 @@ repos:
     rev: v4.5.0
     hooks:
       - id: trailing-whitespace
+        exclude: tests/repositories/fixtures/pypi.org/metadata/.*\.metadata
       - id: end-of-file-fixer
-        exclude: ^.*\.egg-info/
+        exclude: ^.*\.egg-info/|tests/repositories/fixtures/pypi.org/metadata/.*\.metadata
       - id: check-merge-conflict
       - id: check-case-conflict
       - id: check-json

tests/installation/fixtures/with-pypi-repository.test

@@ -65,13 +65,13 @@ files = [
 
 [[package]]
 name = "pytest"
-version = "3.5.0"
+version = "3.5.1"
 description = "pytest: simple powerful testing with Python"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 files = [
-    {file = "pytest-3.5.0-py2.py3-none-any.whl", hash = "sha256:28e4d9c2ae3196d74805c2eba24f350ae4c791a5b9b397c79b41506a48dc64ca"},
-    {file = "pytest-3.5.0.tar.gz", hash = "sha256:677b1d6decd29c041fe64276f29f79fbe66e40c59e445eb251366b4a8ab8bf68"},
+    {file = "pytest-3.5.1-py2.py3-none-any.whl", hash = "sha256:6d3e83b1c1697d220137e436980e73b3ca674f643e666d7c24b0321cb57b76a4"},
+    {file = "pytest-3.5.1.tar.gz", hash = "sha256:b8fe151f3e181801dd38583a1c03818fbc662a8fce96c9063a0af624613e78f8"},
 ]
 
 [package.dependencies]

tests/repositories/fixtures/pypi.org/generate.py

@@ -4,7 +4,7 @@
 
 1. Fetch relevant project json file from https://pypi.org/simple/<name>.
 2. Fetch relevant release json file from https://pypi.org/pypi/<name>/<version>/json.
-3. Download all files (if not otherwise specified) for each release.
+3. Download all files (if not otherwise specified) for each release, including <filename>.metadata files.
 4. Stub (zero-out) all files not relevant for test cases, only sdist and bdist metadata is retained.
     a, We also retain `__init__.py` files as some packages use it for dynamic version detection when building sdist.
     b. Some release bdist, notably that of setuptools, wheel and poetry-core are retained as is in the `dist/` directory
@@ -51,6 +51,7 @@
 from typing import Callable
 from typing import Iterator
 
+from packaging.metadata import parse_email
 from poetry.core.masonry.builders.sdist import SdistBuilder
 from poetry.core.packages.package import Package
 from tests.helpers import FIXTURE_PATH
@@ -64,6 +65,8 @@
 
 
 if TYPE_CHECKING:
+    import requests
+
     from poetry.core.packages.utils.link import Link
 
 logger = logging.getLogger("pypi.generator")
@@ -368,12 +371,25 @@ def known_releases_by_project(self) -> dict[str, list[str]]:
 
         return result
 
+    @cached_property
+    def known_missing_files(self) -> set[str]:
+        return {pkg.name for pkg in PROJECT_SPECS if pkg.ignore_missing_files}
+
+    @staticmethod
+    def _clean_file_item(data: dict[str, Any]) -> dict[str, Any]:
+        filename = data["filename"]
+        metadata_file = (
+            FIXTURE_PATH_REPOSITORIES_PYPI / "metadata" / f"{filename}.metadata"
+        )
+
+        if metadata_file.exists():
+            metadata = ReleaseFileMetadata(metadata_file)
+            for key in ["core-metadata", "data-dist-info-metadata"]:
+                data[key] = {"sha256": metadata.sha256}
+
+        return data
+
     def clean(self) -> None:
-        ignore_missing_files = {
-            project.name
-            for project in self.project_specs
-            if project.ignore_missing_files
-        }
         json_fixture_dir = FIXTURE_PATH_REPOSITORIES_PYPI.joinpath("json")
 
         for file in json_fixture_dir.glob("*.json"):
@@ -392,16 +408,19 @@ def clean(self) -> None:
             existing_content = file.read_text(encoding="utf-8")
             data = json.loads(existing_content)
 
-            if "versions" in data:
+            if "versions" in data and file.stem not in self.known_missing_files:
                 data["versions"] = self.known_releases_by_project[file.stem]
 
-            if "files" in data and package.name not in ignore_missing_files:
+            if "files" in data and package.name not in self.known_missing_files:
                 data["files"] = [
-                    _file
+                    self._clean_file_item(_file)
                     for _file in data["files"]
                     if RELEASE_FILE_COLLECTION.filename_exists(_file["filename"])
                 ]
 
+            if "meta" in data:
+                data["meta"]["_last-serial"] = 0
+
             content = json.dumps(data, ensure_ascii=False, indent=2)
             if existing_content != content:
                 logger.info(
@@ -461,6 +480,8 @@ def add_release_file(self, spec: ReleaseSpecification) -> list[ReleaseFileMetada
         for link in links:
             logger.info("Processing release file %s", link.filename)
 
+            self.process_metadata_file(link)
+
             existing_release_file_location = RELEASE_FILE_LOCATIONS.dist.joinpath(
                 link.filename
             )
@@ -518,6 +539,30 @@ def bdist_check(filename: str) -> bool:
 
         return bdist_check
 
+    def process_metadata_file(self, link: Link) -> None:
+        # we enforce the availability of the metadata file
+        link._metadata = True
+
+        logger.info("Processing metadata file for %s", link.filename)
+
+        assert link.metadata_url is not None
+        response: requests.Response = self.pypi.session.get(
+            link.metadata_url, raise_for_status=False
+        )
+
+        if response.status_code != 200:
+            logger.info("Skipping metadata for %s", link.filename)
+            return None
+
+        metadata, _ = parse_email(response.content)
+        content = response.content.decode(encoding="utf-8").replace(
+            metadata["description"], ""
+        )
+
+        FIXTURE_PATH_REPOSITORIES_PYPI.joinpath(
+            "metadata", f"{link.filename}.metadata"
+        ).write_text(content, encoding="utf-8")
+
     def copy_as_is(self, link: Link) -> ReleaseFileMetadata:
         dst = FIXTURE_PATH_REPOSITORIES_PYPI / "dists" / link.filename
         logger.info(
@@ -715,8 +760,9 @@ def __eq__(self, other: object) -> bool:
 
 
 if __name__ == "__main__":
+    # set refresh to true when recreating package json files
     factory = MockedRepositoryFactory(
-        RELEASE_SPECS, PROJECT_SPECS, PyPiRepository(disable_cache=True), refresh=False
+        RELEASE_SPECS, PROJECT_SPECS, PyPiRepository(disable_cache=True), refresh=True
     )
     factory.process()
     factory.clean()

tests/repositories/fixtures/pypi.org/json/attrs.json

@@ -1,25 +1,44 @@
 {
-  "name": "attrs",
   "files": [
     {
+      "core-metadata": {
+        "sha256": "a1828f9b7a019e96302759189410f380814be1dd57a201a56c078f9e8e11a2e5"
+      },
+      "data-dist-info-metadata": {
+        "sha256": "a1828f9b7a019e96302759189410f380814be1dd57a201a56c078f9e8e11a2e5"
+      },
       "filename": "attrs-17.4.0-py2.py3-none-any.whl",
-      "url": "https://files.pythonhosted.org/packages/b5/60/4e178c1e790fd60f1229a9b3cb2f8bc2f4cc6ff2c8838054c142c70b5adc/attrs-17.4.0-py2.py3-none-any.whl",
       "hashes": {
         "md5": "9d32f2b5a93343e01f54d87740f2da60",
         "sha256": "d38e57f381e891928357c68e300d28d3d4dcddc50486d5f8dfaf743d40477619"
-      }
+      },
+      "requires-python": null,
+      "size": 31658,
+      "upload-time": "2017-12-30T08:20:05.582456Z",
+      "url": "https://files.pythonhosted.org/packages/b5/60/4e178c1e790fd60f1229a9b3cb2f8bc2f4cc6ff2c8838054c142c70b5adc/attrs-17.4.0-py2.py3-none-any.whl",
+      "yanked": false
     },
     {
+      "core-metadata": false,
+      "data-dist-info-metadata": false,
       "filename": "attrs-17.4.0.tar.gz",
-      "url": "https://files.pythonhosted.org/packages/8b/0b/a06cfcb69d0cb004fde8bc6f0fd192d96d565d1b8aa2829f0f20adb796e5/attrs-17.4.0.tar.gz",
       "hashes": {
         "md5": "c03e5b3608d9071fbd098850d8922668",
         "sha256": "eb7536a1e6928190b3008c5b350bdf9850d619fff212341cd096f87a27a5e564"
-      }
+      },
+      "requires-python": null,
+      "size": 97071,
+      "upload-time": "2017-12-30T08:20:08.575620Z",
+      "url": "https://files.pythonhosted.org/packages/8b/0b/a06cfcb69d0cb004fde8bc6f0fd192d96d565d1b8aa2829f0f20adb796e5/attrs-17.4.0.tar.gz",
+      "yanked": false
     }
   ],
   "meta": {
-    "api-version": "1.0",
-    "_last-serial": 3451237
-  }
+    "_last-serial": 0,
+    "api-version": "1.1"
+  },
+  "name": "attrs",
+  "versions": [
+    "17.4.0"
+  ]
 }

tests/repositories/fixtures/pypi.org/json/black.json

@@ -1,49 +1,77 @@
 {
   "files": [
     {
+      "core-metadata": {
+        "sha256": "256c27c1f5cc5c7bad6049ff2c339ac9419f5fc00eddbb895cf4780d7b6752b3"
+      },
+      "data-dist-info-metadata": {
+        "sha256": "256c27c1f5cc5c7bad6049ff2c339ac9419f5fc00eddbb895cf4780d7b6752b3"
+      },
       "filename": "black-19.10b0-py36-none-any.whl",
       "hashes": {
         "md5": "4a420234749e1ea350581160ef51cd02",
         "sha256": "3471ff321348d851b6f3047f4ed42c88622ac038caf7c2d160bd653334c88e88"
       },
       "requires-python": ">=3.6",
+      "size": 97525,
+      "upload-time": "2019-10-28T23:53:54.000711Z",
       "url": "https://files.pythonhosted.org/packages/fd/bb/ad34bbc93d1bea3de086d7c59e528d4a503ac8fe318bd1fa48605584c3d2/black-19.10b0-py36-none-any.whl",
       "yanked": false
     },
     {
+      "core-metadata": false,
+      "data-dist-info-metadata": false,
       "filename": "black-19.10b0.tar.gz",
       "hashes": {
         "md5": "c383543109a66a5a99113e6326db5251",
         "sha256": "6cada614d5d2132698c6d5fff384657273d922c4fffa6a2f0de9e03e25b8913a"
       },
       "requires-python": ">=3.6",
+      "size": 1019740,
+      "upload-time": "2019-10-28T23:54:05.455213Z",
       "url": "https://files.pythonhosted.org/packages/b0/dc/ecd83b973fb7b82c34d828aad621a6e5865764d52375b8ac1d7a45e23c8d/black-19.10b0.tar.gz",
       "yanked": false
     },
     {
+      "core-metadata": {
+        "sha256": "6b5b5209d6862dde7399150665655faa4526565c775f125ad004746fe96d4d4c"
+      },
+      "data-dist-info-metadata": {
+        "sha256": "6b5b5209d6862dde7399150665655faa4526565c775f125ad004746fe96d4d4c"
+      },
       "filename": "black-21.11b0-py3-none-any.whl",
       "hashes": {
         "md5": "294e105f34e2e21286a49bfcfb8fb6ae",
         "sha256": "e16b6879ed61f9268994b879174fad1cb2319a651afd20f8cf036428ac65f846"
       },
       "requires-python": ">=3.6.2",
+      "size": 155131,
+      "upload-time": "2021-11-17T02:32:14.551680Z",
       "url": "https://files.pythonhosted.org/packages/3d/ad/1cf514e7f9ee4c3d8df7c839d7977f7605ad76557f3fca741ec67f76dba6/black-21.11b0-py3-none-any.whl",
       "yanked": "Broken regex dependency. Use 21.11b1 instead."
     },
     {
+      "core-metadata": false,
+      "data-dist-info-metadata": false,
       "filename": "black-21.11b0.tar.gz",
       "hashes": {
         "md5": "f01267bf2613f825dd6684629c1c829e",
         "sha256": "f23c482185d842e2f19d506e55c004061167e3c677c063ecd721042c62086ada"
       },
       "requires-python": ">=3.6.2",
+      "size": 593164,
+      "upload-time": "2021-11-17T02:32:16.396821Z",
       "url": "https://files.pythonhosted.org/packages/2f/db/03e8cef689ab0ff857576ee2ee288d1ff2110ef7f3a77cac62e61f18acaf/black-21.11b0.tar.gz",
       "yanked": "Broken regex dependency. Use 21.11b1 instead."
     }
   ],
   "meta": {
-    "_last-serial": 14955312,
-    "api-version": "1.0"
+    "_last-serial": 0,
+    "api-version": "1.1"
   },
-  "name": "black"
+  "name": "black",
+  "versions": [
+    "19.10b0",
+    "21.11b0"
+  ]
 }

tests/repositories/fixtures/pypi.org/json/cleo.json

@@ -2,10 +2,10 @@
   "files": [
     {
       "core-metadata": {
-        "sha256": "892426a14eeae04120b2baa0212d7090f3a22e64ffd5d0ad3bc4c515b59a9c89"
+        "sha256": "5627f48cfca57f878bf73ea222fa8ca2f24ad248061a2f0151f22219386838f9"
       },
       "data-dist-info-metadata": {
-        "sha256": "892426a14eeae04120b2baa0212d7090f3a22e64ffd5d0ad3bc4c515b59a9c89"
+        "sha256": "5627f48cfca57f878bf73ea222fa8ca2f24ad248061a2f0151f22219386838f9"
       },
       "filename": "cleo-1.0.0a5-py3-none-any.whl",
       "hashes": {
@@ -34,7 +34,7 @@
     }
   ],
   "meta": {
-    "_last-serial": 20402192,
+    "_last-serial": 0,
     "api-version": "1.1"
   },
   "name": "cleo",

tests/repositories/fixtures/pypi.org/json/clikit.json

@@ -1,27 +1,44 @@
 {
-  "name": "clikit",
   "files": [
     {
+      "core-metadata": {
+        "sha256": "bf1c55d7159f7b783967c3035898bcf4fee282a654fb70efd0fdafc6e782f8b8"
+      },
+      "data-dist-info-metadata": {
+        "sha256": "bf1c55d7159f7b783967c3035898bcf4fee282a654fb70efd0fdafc6e782f8b8"
+      },
       "filename": "clikit-0.2.4-py2.py3-none-any.whl",
-      "url": "https://files.pythonhosted.org/packages/7b/0d/bb4c8a2d0edca8c300373ed736fb4680cf73be5be2ff84544dee5f979c14/clikit-0.2.4-py2.py3-none-any.whl",
       "hashes": {
         "md5": "c3558fef2a1148bb1df96376def5c8fe",
         "sha256": "60900adbac91d6d2cefc88efe2639ce3090f4520106596ac855bee3763f276c0"
       },
-      "requires-python": ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+      "requires-python": ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*",
+      "size": 85786,
+      "upload-time": "2019-05-11T17:09:23.516387Z",
+      "url": "https://files.pythonhosted.org/packages/7b/0d/bb4c8a2d0edca8c300373ed736fb4680cf73be5be2ff84544dee5f979c14/clikit-0.2.4-py2.py3-none-any.whl",
+      "yanked": false
     },
     {
+      "core-metadata": false,
+      "data-dist-info-metadata": false,
       "filename": "clikit-0.2.4.tar.gz",
-      "url": "https://files.pythonhosted.org/packages/c5/33/14fad4c82f256b0ef60dd25d4b6d8145b463da5274fd9cd842f06af318ed/clikit-0.2.4.tar.gz",
       "hashes": {
         "md5": "f7cdbad3508038a04561f646aae68146",
         "sha256": "0fdd41e86e8b118a8b1e94ef2835925ada541d481c9b3b2fc635fa68713e6125"
       },
-      "requires-python": ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+      "requires-python": ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*",
+      "size": 50980,
+      "upload-time": "2019-05-11T17:09:25.865051Z",
+      "url": "https://files.pythonhosted.org/packages/c5/33/14fad4c82f256b0ef60dd25d4b6d8145b463da5274fd9cd842f06af318ed/clikit-0.2.4.tar.gz",
+      "yanked": false
     }
   ],
   "meta": {
-    "api-version": "1.0",
-    "_last-serial": 5256718
-  }
+    "_last-serial": 0,
+    "api-version": "1.1"
+  },
+  "name": "clikit",
+  "versions": [
+    "0.2.4"
+  ]
 }