misc: let's make copr_new_packages a lot faster

misc/copr_new_packages.py

@@ -9,6 +9,7 @@
 import argparse
 from datetime import date, timedelta
 from copr.v3 import Client
+from tqdm import tqdm
 
 
 # @TODO Check for package review RHBZ and print warnings.
@@ -33,36 +34,44 @@ def pick_project_candidates(client, projects, since):
     Magazine article). By such projects we consider those with at least one
     succeeded build and at least some project information filled.
     """
+    rawhide_pks_resp = subprocess.run(
+        ["dnf", "--quiet", "--repo=fedora", "repoquery", "--queryformat", "%{name}\n", "*"],
+        stdout=subprocess.PIPE,
+    )
+    fedora_rawhide_pkgs = {x for x in rawhide_pks_resp.stdout.decode().split()}
+
     picked = []
-    for project in projects:
+    for project in tqdm(projects):
         if project.unlisted_on_hp:
-            print("Skipping {}, it is unlisted on Copr homepage".format(project.full_name))
+            tqdm.write("Skipping {}, it is unlisted on Copr homepage".format(project.full_name))
             continue
 
         if not any([project.description, project.instructions,
                     project.homepage, project.contact]):
-            print("Skipping {}, it has no information filled in".format(project.full_name))
+            tqdm.write("Skipping {}, it has no information filled in".format(project.full_name))
             continue
 
         builds = client.build_proxy.get_list(project.ownername, project.name)
         if not builds:
-            print("Skipping {}, no builds".format(project.full_name))
+            tqdm.write("Skipping {}, no builds".format(project.full_name))
             continue
 
         builds = [b for b in builds if b.state == "succeeded"]
         if not builds:
-            print("Skipping {}, no succeeded builds".format(project.full_name))
+            tqdm.write("Skipping {}, no succeeded builds".format(project.full_name))
             continue
 
         builds = filter_unique_package_builds(builds)
-        builds = [b for b in builds if not is_in_fedora(b.source_package["name"])]
+        builds = [
+            b for b in builds if b.source_package["name"] not in fedora_rawhide_pkgs
+        ]
         if not builds:
-            print("Skipping {}, all packages already in Fedora".format(project.full_name))
+            tqdm.write("Skipping {}, all packages already in Fedora".format(project.full_name))
             continue
 
         started_on = date.fromtimestamp(builds[-1].started_on)
         if started_on < since:
-            print("Reached older project than {}, ending with {}".format(since, project.full_name))
+            tqdm.write("Reached older project than {}, ending with {}".format(since, project.full_name))
             break
 
         picked.append((project, builds))
@@ -81,14 +90,6 @@ def filter_unique_package_builds(builds):
     return unique.values()
 
 
-def is_in_fedora(packagename):
-    """
-    Check if a given package is already provided by Fedora official repositories
-    """
-    cmd = ["koji", "search", "package", packagename]
-    return bool(subprocess.check_output(cmd))
-
-
 def get_parser():
     description = ("This tool lists new packages in Copr, that are not yet "
                    "available in Fedora. It's main purpose is to help us pick "
@@ -102,6 +103,13 @@ def get_parser():
         type=date.fromisoformat,
         help="Search for new projects since YYYY-MM-DD",
     )
+    parser.add_argument(
+        "--limit",
+        required=False,
+        default=1000,
+        type=int,
+        help="Limit the number of projects to be checked",
+    )
     return parser
 
 
@@ -110,7 +118,7 @@ def main():
     args = parser.parse_args()
 
     client = Client.create_from_config_file()
-    projects = get_new_projects(client)
+    projects = get_new_projects(client, args.limit)
 
     print("Going to filter interesting projects since {}".format(args.since))
     print("This may take a while, ...")