add back literate and strip svg for all nbs (#269)

* add back literate and strip svg for all nbs

* fix

* also list jl files

* fix

* fix

* fix

* fix

* fix

* TOC

.github/workflows/ci.yml

@@ -11,6 +11,10 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  ALLOWERRORS: false
+  NBCACHE: ".cache"
+
 jobs:
   setup:
     permissions:
@@ -40,7 +44,7 @@ jobs:
         docker push ${IMG}
     - name: List notebooks as a JSON array
       id: set-matrix
-      run: echo "matrix=$(python -c 'import glob, json; print(json.dumps(glob.glob("**/*.ipynb", root_dir="docs", recursive=True)))')" >> "$GITHUB_OUTPUT"
+      run: echo "matrix=$(python -c 'import glob, json; print(json.dumps(glob.glob("**/*.ipynb", root_dir="docs", recursive=True) + glob.glob("**/*.jl", root_dir="docs", recursive=True)))')" >> "$GITHUB_OUTPUT"
 
   execute:
     needs: setup
@@ -57,43 +61,41 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Cache notebook
-        uses: actions/cache/restore@v4
+        uses: actions/cache@v4
         id: nb-cache
         with:
-          path: ${{ env.NB }}
-          key: notebook-${{ needs.setup.outputs.hash }}-${{ hashFiles(env.NB) }}
+          path: ${{ env.NBCACHE }}
+          key: ${{ needs.setup.outputs.ver }}-${{ needs.setup.outputs.hash }}-${{ hashFiles(env.NB) }}
       - name: Pull docker image
         if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
         run: |
           docker pull ${{ env.IMG }}
           docker images ${{ env.IMG }}
-      - name: Get Julia kernel name
-        id: jl-kernel
-        if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
-        run: docker run ${{ env.IMG }} julia -e 'print("name=--ExecutePreprocessor.kernel_name=julia-1.", VERSION.minor)' >> "$GITHUB_OUTPUT"
       - name: Execute notebook
         if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
         run: >
-          docker run -w /app -v ${{ github.workspace }}:/app
+          docker run -w /tmp -v ${{ github.workspace }}:/tmp
+          -e NB=${{ env.NB }}
+          -e NBCACHE=${{ env.NBCACHE }}
+          -e WORKSPACE=${{ github.workspace }}
+          -e ALLOWERRORS=${{ env.ALLOWERRORS }}
           ${{ env.IMG }}
-          jupyter nbconvert --to notebook --execute --inplace --ExecutePreprocessor.timeout=-1 ${{ steps.jl-kernel.outputs.name }} ${{ env.NB }}
+          julia --project=@. ci.jl
+      - name: Claim output cache folder
+        run: |
+          sudo chown -R $USER ${{ env.NBCACHE }}
+          ls -R ${{ env.NBCACHE }}
       - name: Claim output notebook
         if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
-        run: sudo chown $USER ${{ env.NB }}
-      - name: Cache notebook
-        uses: actions/cache/save@v4
-        if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }}
-        with:
-          path: ${{ env.NB }}
-          key: ${{ steps.nb-cache.outputs.cache-primary-key }}
+        run: sudo chown -R $USER ${{ env.NBCACHE }}
       - name: Convert artifact Name
         id: art
         run: echo "name=$(echo ${{ env.NB }} | sed 's/\//-/g')" >> "$GITHUB_OUTPUT"
       - name: Upload Notebook
         uses: actions/upload-artifact@v4
         with:
-          name: notebook-${{ needs.setup.outputs.hash }}-${{ hashFiles(env.NB) }}
-          path: docs/*${{ matrix.notebook }}
+          name: notebook-${{ steps.art.outputs.name }}-${{ needs.setup.outputs.hash }}-${{ hashFiles(env.NB) }}
+          path: ${{ env.NBCACHE }}
           include-hidden-files: true
           retention-days: 1
 
@@ -106,11 +108,11 @@ jobs:
       - name: Download notebooks
         uses: actions/download-artifact@v4
         with:
-          path: out/
+          path: ${{ env.NBCACHE }}/
           pattern: notebook-*
           merge-multiple: true
       - name: Copy back built notebooks
-        run: cp --verbose -rf out/* docs/
+        run: cp --verbose -rf ${{ env.NBCACHE }}/docs/* docs/
       - name: Setup Python
         uses: actions/setup-python@v5
         id: setup-python

ci.jl

@@ -0,0 +1,63 @@
+using Literate
+using JSON
+using Pkg
+using IJulia
+
+ENV["GKSwstype"] = "100"
+
+function main(; rmsvg=true)
+    file = get(ENV, "NB", "test.ipynb")
+    cachedir = get(ENV, "NBCACHE", ".cache")
+    nb = if endswith(file, ".jl")
+        run_literate(file; cachedir)
+    elseif endswith(file, ".ipynb")
+        IJulia.installkernel("Julia", "--project=@.")
+        run_ipynb(file; cachedir)
+    else
+        error("$(file) is not a valid notebook file!")
+    end
+    rmsvg && strip_svg(nb)
+    return nothing
+end
+
+# Strip SVG output from a Jupyter notebook
+function strip_svg(ipynb)
+    @info "Stripping SVG in $(ipynb)"
+    nb = open(JSON.parse, ipynb, "r")
+    for cell in nb["cells"]
+        !haskey(cell, "outputs") && continue
+        for output in cell["outputs"]
+            !haskey(output, "data") && continue
+            datadict = output["data"]
+            if haskey(datadict, "image/png") || haskey(datadict, "image/jpeg")
+                delete!(datadict, "text/html")
+                delete!(datadict, "image/svg+xml")
+            end
+        end
+    end
+    rm(ipynb)
+    open(ipynb, "w") do io
+        JSON.print(io, nb, 1)
+    end
+    return ipynb
+end
+
+function run_literate(file; cachedir = ".cache")
+    outpath = joinpath(abspath(pwd()), cachedir, dirname(file))
+    mkpath(outpath)
+    ipynb = Literate.notebook(file, outpath; mdstrings=true, execute=true)
+    return ipynb
+end
+
+function run_ipynb(file; cachedir = ".cache")
+    outpath = joinpath(abspath(pwd()), cachedir, file)
+    mkpath(dirname(outpath))
+    kernelname = "--ExecutePreprocessor.kernel_name=julia-1.$(VERSION.minor)"
+    execute = get(ENV, "ALLOWERRORS", " ") == "true" ? "--execute --allow-errors" : "--execute"
+    timeout = "--ExecutePreprocessor.timeout=" * get(ENV, "TIMEOUT", "-1")
+    cmd = `jupyter nbconvert --to notebook $(execute) $(timeout) $(kernelname) --output $(outpath) $(file)`
+    run(cmd)
+    return outpath
+end
+
+main()

docs/_toc.yml

@@ -5,4 +5,6 @@ format: jb-book
 root: index
 chapters:
   - file: pyplot
+  - file: pythonplot
   - file: sub/plots
+  - file: sub/plots-lit

docs/pyplot.ipynb

@@ -40,6 +40,14 @@
     "plt.plot(1:5, rand(1:6, 5))"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "e5ab6ecb",
+   "metadata": {},
+   "source": [
+    "## Runtime information"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

docs/pythonplot.jl

@@ -0,0 +1,21 @@
+#===
+# Plotting with PythonPlot.jl
+
+Using Literate.jl
+===#
+import PythonPlot as plt
+using Random
+Random.seed!(2022)
+
+#---
+plt.figure()
+plt.plot(1:5, rand(1:6, 5))
+plt.gcf()
+
+# ## Runtime information
+import Pkg
+Pkg.status()
+
+#---
+import InteractiveUtils
+InteractiveUtils.versioninfo()

docs/sub/plots-lit.jl

@@ -0,0 +1,19 @@
+#===
+# Plotting by Plots.jl
+
+Using Literate.jl
+===#
+using Plots
+using Random
+Random.seed!(2022)
+
+#---
+plot(rand(1:6, 5))
+
+# ## Runtime information
+import Pkg
+Pkg.status()
+
+#---
+import InteractiveUtils
+InteractiveUtils.versioninfo()

docs/sub/plots.ipynb

@@ -41,6 +41,14 @@
     "plot(1:5, rand(1:6, 5))"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "08724089",
+   "metadata": {},
+   "source": [
+    "## Runtime information"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

env.Dockerfile

@@ -20,5 +20,5 @@ RUN uv pip install --system --no-cache nbconvert -r requirements.txt
 # Julia dependencies
 COPY Project.toml Manifest.toml ./
 COPY src/ src
-RUN julia --color=yes -e 'using Pkg; Pkg.add("IJulia"); import IJulia; IJulia.installkernel("Julia", "--project=@.")' && \
+RUN julia --color=yes -e 'using Pkg; Pkg.add(["IJulia", "Literate", "JSON"]); import IJulia; IJulia.installkernel("Julia", "--project=@.")' && \
     julia --color=yes --project=@. -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()'