Use one artifact for each deposited GAP package (#1069)

Having one artifact per package instead of one single artifact has the
advantage that user's don't need to download a single 500 MB blob (which
is very annoying if one is on a flaky connection). And when updating
GAP, then only those package artifacts that actually changed need to be
re-downloaded, which reduces disk space usage.

This also paves the way for shipping fewer packages out of the box (e.g.
we could make some or all of these artifacts lazy).

Moreover the `GAP_pkg_XYZ.jl` wrapper packages use the same artifacts,
so if one uses those no extra disk or download overhead is incurred.

The packages used here are from GAP 4.14.0. The `Artifacts.toml` file
can be updated for future GAP releases by using the helper script
`etc/update_artifacts.jl`.

CHANGES.md

@@ -3,6 +3,8 @@
 ## Version 0.13.0-DEV (released YYYY-MM-DD)
 
 - Update to GAP 4.14.0
+- Instead of downloading a single huge "artifact" containing all deposited GAP
+  packages, we now use (and download) one artifact per GAP package.
 
 ## Version 0.12.1 (released 2024-12-09)
 

Project.toml

@@ -21,6 +21,7 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Scratch = "6c6a2e73-6563-6170-7368-637461726353"
+TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 
 [compat]
 AbstractAlgebra = "0.41.11, 0.42.1, 0.43"
@@ -40,4 +41,5 @@ Pkg = "1.6"
 REPL = "1.6"
 Random = "1.6"
 Scratch = "1.1"
+TOML = "<0.0.1, 1"
 julia = "1.6"

etc/Project.toml

@@ -0,0 +1,8 @@
+[deps]
+ArtifactUtils = "8b73e784-e7d8-4ea5-973d-377fed4e3bce"
+Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
+Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+GZip = "92fee26a-97fe-5a0c-ad85-20a5f3185b63"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"

etc/README.md

@@ -77,14 +77,8 @@ For this to work, follow these instructions:
 5. This opens a Julia session with the override in effect. You can now e.g. load GAP.jl
    via `using GAP`, or install other packages (such as Oscar) and test with them.
 
-## Updating the package tarball
-
-```
-using ArtifactUtils
-add_artifact!(
-  "Artifacts.toml",
-  "gap_packages",
-  "https://github.com/gap-system/PackageDistro/releases/download/v4.13.1/packages.tar.gz";
-  force=true
-)
-```
+## Updating the package artifacts
+
+Run this from the root director of GAP.jl:
+
+    julia --project=etc etc/update_artifacts.jl 4.X.Y

etc/update_artifacts.jl

@@ -0,0 +1,132 @@
+#
+# This script is used to update Artifacts.toml
+#
+# Usage variants:
+#   julia --project=etc etc/update_artifacts.jl 4.14.0
+#   julia --project=etc etc/update_artifacts.jl https://.../package-infos.json
+#   julia --project=etc etc/update_artifacts.jl https://.../package-infos.json.gz
+#   julia --project=etc etc/update_artifacts.jl local/path/package-infos.json
+#   julia --project=etc etc/update_artifacts.jl local/path/package-infos.json.gz
+#
+
+using Downloads: download
+import Pkg
+using Pkg.Artifacts
+import Pkg.PlatformEngines
+#using Pkg.GitTools
+import GZip
+import JSON
+import TOML
+import SHA
+
+
+function sha256sum(tarball_path)
+    return open(tarball_path, "r") do io
+        return bytes2hex(SHA.sha256(io))
+    end
+end
+
+function add_artifacts_for_packages(; pkginfos_path::String = "package-infos.json", artifacts_toml::String="Artifacts.toml")
+    pkgs = GZip.open(JSON.parse, pkginfos_path, "r")
+    artifacts = TOML.parsefile(artifacts_toml)
+
+    for name in sort(collect(keys(pkgs)))
+        print("Processing '$name' ")
+        pkginfo = pkgs[name]
+        add_artifacts_for_package(pkginfo, artifacts)
+
+        # write it all out again
+        open(artifacts_toml, "w") do io
+            TOML.print(io, artifacts; sorted=true)
+        end
+
+    end
+
+    # delete artifacts for any packages that are no longer distributed with GAP
+    pkg_names = ["GAP_pkg_"*lowercase(pkginfo["PackageName"]) for (name, pkginfo) in pkgs]
+    to_be_removed = setdiff(keys(artifacts), pkg_names)
+    for name in to_be_removed
+        delete!(artifacts, name)
+    end
+
+    # write it all out again
+    open(artifacts_toml, "w") do io
+        TOML.print(io, artifacts; sorted=true)
+    end
+
+    return nothing
+end
+
+function add_artifacts_for_package(pkginfo, artifacts)
+    gap_pkgname = pkginfo["PackageName"]
+    pkgname = lowercase(gap_pkgname)
+    artifact_name = "GAP_pkg_$(pkgname)"
+
+    #
+    # extract info about the package tarball
+    #
+    sha256 = pkginfo["ArchiveSHA256"]
+    url = pkginfo["ArchiveURL"]
+    formats = split(pkginfo["ArchiveFormats"], " ")
+    url *= first(formats)  # this matches what the PackageDistro does, and allow us to use ArchiveSHA256
+    url2 = "https://files.gap-system.org/pkg/" * basename(url)
+
+    # check if this file is already registered
+    if haskey(artifacts, artifact_name)
+        downloads = artifacts[artifact_name]["download"]
+        d = Dict("sha256" => sha256, "url" => url)
+        if d in downloads
+            println("  already present")
+            d2 = Dict("sha256" => sha256, "url" => url2)
+            if !(d2 in downloads)
+                println("  added backup URL $(url2)")
+                push!(downloads, d2)
+            end
+            return
+        end
+    end
+
+    # add the artifact
+    println("  importing new archive $url")
+    tarball_path = download(url)
+    tarball_hash = sha256sum(tarball_path)
+    if sha256 != tarball_hash
+        error("SHA256 mismatch for $url")
+    end
+
+    git_tree_sha1 = create_artifact() do artifact_dir
+        Pkg.PlatformEngines.unpack(tarball_path, artifact_dir)
+    end
+
+    rm(tarball_path)
+    #clear && remove_artifact(git_tree_sha1)
+
+    artifacts[artifact_name] = Dict{String,Any}(
+        "git-tree-sha1" => bytes2hex(git_tree_sha1.bytes),
+        "download" => [ Dict("sha256" => sha256, "url" => url),
+                        Dict("sha256" => sha256, "url" => url2) ]
+    )
+
+    return
+end
+
+# https://github.com/gap-system/gap/releases/download/v4.14.0/package-infos.json.gz
+# https://github.com/gap-system/gap/releases/download/v4.14.0/package-infos.json.gz.sha256
+
+if length(ARGS) > 0
+    desc = ARGS[1]
+    if startswith(desc, "http")
+        pkginfos_url = desc
+        println("Download package-infos from $(pkginfos_url)")
+        pkginfos_path = download(pkginfos_url)
+    elseif startswith(desc, "4.")
+        pkginfos_url = "https://github.com/gap-system/gap/releases/download/v$desc/package-infos.json.gz"
+        println("Download package-infos from $(pkginfos_url)")
+        pkginfos_path = download(pkginfos_url)
+    else
+        pkginfos_path = desc
+    end
+
+    println("processing $(pkginfos_path)")
+    add_artifacts_for_packages(; pkginfos_path)
+end

src/setup.jl

@@ -1,16 +1,17 @@
 module Setup
 
 using Pkg: GitTools
-import Artifacts: @artifact_str
+import Artifacts: find_artifacts_toml, @artifact_str
 import GAP_jll
 import GAP_lib_jll
 import GAP_pkg_juliainterface_jll
 import Scratch: @get_scratch!
 import Pidfile
+import TOML
 
 # to separate the scratchspaces of different GAP.jl copies and Julia versions
 # put the Julia version and the hash of the path to this file into the key
-const scratch_key = "gap_$(string(hash(@__FILE__)))_$(VERSION.major).$(VERSION.minor)"
+const scratch_key = "gap_$(hash(@__FILE__))-$(VERSION.major).$(VERSION.minor)"
 
 gaproot() = @get_scratch!(scratch_key)
 
@@ -195,8 +196,12 @@ function regenerate_gaproot()
         force_symlink("../../gac",
                       joinpath(gaproot_mutable, "bin", sysinfo["GAParch"], "gac"))
 
-        # create a `pkg` symlink to the GAP packages artifact
-        force_symlink(artifact"gap_packages", "$gaproot_mutable/pkg")
+        # create a `pkg` directory with symlinks to all the GAP packages artifacts
+        mkpath(joinpath(gaproot_mutable, "pkg"))
+        pkg_artifacts = filter(startswith("GAP_pkg_"), keys(TOML.parsefile(find_artifacts_toml(@__FILE__))))
+        for name in pkg_artifacts
+            force_symlink(@artifact_str(name), joinpath(gaproot_mutable, "pkg", name))
+        end
 
     end # mkpidlock