From 12c43803400763f6207b1346ca81f95662c67178 Mon Sep 17 00:00:00 2001 From: Jiri Kyjovsky Date: Mon, 11 Nov 2024 16:38:16 +0100 Subject: [PATCH 1/2] misc: copr_new_packages, drop requests to koji api, use repoquery Asking koji for each package was slow, also we were spamming koji API. This gets every package just once. --- misc/copr_new_packages.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/misc/copr_new_packages.py b/misc/copr_new_packages.py index 7b4e945ce..b2412582b 100644 --- a/misc/copr_new_packages.py +++ b/misc/copr_new_packages.py @@ -33,6 +33,12 @@ def pick_project_candidates(client, projects, since): Magazine article). By such projects we consider those with at least one succeeded build and at least some project information filled. """ + rawhide_pks_resp = subprocess.run( + ["dnf", "repoquery", "rawhide", "--queryformat", "%{name}", "*"], + stdout=subprocess.PIPE, + ) + fedora_rawhide_pkgs = {x for x in rawhide_pks_resp.stdout.decode().split("\n")} + picked = [] for project in projects: if project.unlisted_on_hp: @@ -55,7 +61,9 @@ def pick_project_candidates(client, projects, since): continue builds = filter_unique_package_builds(builds) - builds = [b for b in builds if not is_in_fedora(b.source_package["name"])] + builds = [ + b for b in builds if b.source_package["name"] not in fedora_rawhide_pkgs + ] if not builds: print("Skipping {}, all packages already in Fedora".format(project.full_name)) continue @@ -81,14 +89,6 @@ def filter_unique_package_builds(builds): return unique.values() -def is_in_fedora(packagename): - """ - Check if a given package is already provided by Fedora official repositories - """ - cmd = ["koji", "search", "package", packagename] - return bool(subprocess.check_output(cmd)) - - def get_parser(): description = ("This tool lists new packages in Copr, that are not yet " "available in Fedora. It's main purpose is to help us pick " From c20783536a39ce4762666c9d11f9d1d6c863ed5b Mon Sep 17 00:00:00 2001 From: Jiri Kyjovsky Date: Mon, 11 Nov 2024 16:52:59 +0100 Subject: [PATCH 2/2] mics: copr_new_packages add limit option and progressbar --- misc/copr_new_packages.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/misc/copr_new_packages.py b/misc/copr_new_packages.py index b2412582b..1a71af4e5 100644 --- a/misc/copr_new_packages.py +++ b/misc/copr_new_packages.py @@ -9,6 +9,7 @@ import argparse from datetime import date, timedelta from copr.v3 import Client +from tqdm import tqdm # @TODO Check for package review RHBZ and print warnings. @@ -34,30 +35,30 @@ def pick_project_candidates(client, projects, since): succeeded build and at least some project information filled. """ rawhide_pks_resp = subprocess.run( - ["dnf", "repoquery", "rawhide", "--queryformat", "%{name}", "*"], + ["dnf", "--quiet", "--repo=fedora", "repoquery", "--queryformat", "%{name}\n", "*"], stdout=subprocess.PIPE, ) - fedora_rawhide_pkgs = {x for x in rawhide_pks_resp.stdout.decode().split("\n")} + fedora_rawhide_pkgs = {x for x in rawhide_pks_resp.stdout.decode().split()} picked = [] - for project in projects: + for project in tqdm(projects): if project.unlisted_on_hp: - print("Skipping {}, it is unlisted on Copr homepage".format(project.full_name)) + tqdm.write("Skipping {}, it is unlisted on Copr homepage".format(project.full_name)) continue if not any([project.description, project.instructions, project.homepage, project.contact]): - print("Skipping {}, it has no information filled in".format(project.full_name)) + tqdm.write("Skipping {}, it has no information filled in".format(project.full_name)) continue builds = client.build_proxy.get_list(project.ownername, project.name) if not builds: - print("Skipping {}, no builds".format(project.full_name)) + tqdm.write("Skipping {}, no builds".format(project.full_name)) continue builds = [b for b in builds if b.state == "succeeded"] if not builds: - print("Skipping {}, no succeeded builds".format(project.full_name)) + tqdm.write("Skipping {}, no succeeded builds".format(project.full_name)) continue builds = filter_unique_package_builds(builds) @@ -65,12 +66,12 @@ def pick_project_candidates(client, projects, since): b for b in builds if b.source_package["name"] not in fedora_rawhide_pkgs ] if not builds: - print("Skipping {}, all packages already in Fedora".format(project.full_name)) + tqdm.write("Skipping {}, all packages already in Fedora".format(project.full_name)) continue started_on = date.fromtimestamp(builds[-1].started_on) if started_on < since: - print("Reached older project than {}, ending with {}".format(since, project.full_name)) + tqdm.write("Reached older project than {}, ending with {}".format(since, project.full_name)) break picked.append((project, builds)) @@ -102,6 +103,13 @@ def get_parser(): type=date.fromisoformat, help="Search for new projects since YYYY-MM-DD", ) + parser.add_argument( + "--limit", + required=False, + default=1000, + type=int, + help="Limit the number of projects to be checked", + ) return parser @@ -110,7 +118,7 @@ def main(): args = parser.parse_args() client = Client.create_from_config_file() - projects = get_new_projects(client) + projects = get_new_projects(client, args.limit) print("Going to filter interesting projects since {}".format(args.since)) print("This may take a while, ...")