From 4633e56b975edd5bc0907edc47d2e6323a7da81a Mon Sep 17 00:00:00 2001 From: Wen Wei Tseng Date: Tue, 22 Oct 2024 23:53:09 +0800 Subject: [PATCH] add back literate and strip svg for all nbs (#269) * add back literate and strip svg for all nbs * fix * also list jl files * fix * fix * fix * fix * fix * TOC --- .github/workflows/ci.yml | 44 ++++++++++++++-------------- ci.jl | 63 ++++++++++++++++++++++++++++++++++++++++ docs/_toc.yml | 2 ++ docs/pyplot.ipynb | 8 +++++ docs/pythonplot.jl | 21 ++++++++++++++ docs/sub/plots-lit.jl | 19 ++++++++++++ docs/sub/plots.ipynb | 8 +++++ env.Dockerfile | 2 +- 8 files changed, 145 insertions(+), 22 deletions(-) create mode 100644 ci.jl create mode 100644 docs/pythonplot.jl create mode 100644 docs/sub/plots-lit.jl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4bd7e0..9ae7ac7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + ALLOWERRORS: false + NBCACHE: ".cache" + jobs: setup: permissions: @@ -40,7 +44,7 @@ jobs: docker push ${IMG} - name: List notebooks as a JSON array id: set-matrix - run: echo "matrix=$(python -c 'import glob, json; print(json.dumps(glob.glob("**/*.ipynb", root_dir="docs", recursive=True)))')" >> "$GITHUB_OUTPUT" + run: echo "matrix=$(python -c 'import glob, json; print(json.dumps(glob.glob("**/*.ipynb", root_dir="docs", recursive=True) + glob.glob("**/*.jl", root_dir="docs", recursive=True)))')" >> "$GITHUB_OUTPUT" execute: needs: setup @@ -57,43 +61,41 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: Cache notebook - uses: actions/cache/restore@v4 + uses: actions/cache@v4 id: nb-cache with: - path: ${{ env.NB }} - key: notebook-${{ needs.setup.outputs.hash }}-${{ hashFiles(env.NB) }} + path: ${{ env.NBCACHE }} + key: ${{ needs.setup.outputs.ver }}-${{ needs.setup.outputs.hash }}-${{ hashFiles(env.NB) }} - name: Pull docker image if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }} run: | docker pull ${{ env.IMG }} docker images ${{ env.IMG }} - - name: Get Julia kernel name - id: jl-kernel - if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }} - run: docker run ${{ env.IMG }} julia -e 'print("name=--ExecutePreprocessor.kernel_name=julia-1.", VERSION.minor)' >> "$GITHUB_OUTPUT" - name: Execute notebook if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }} run: > - docker run -w /app -v ${{ github.workspace }}:/app + docker run -w /tmp -v ${{ github.workspace }}:/tmp + -e NB=${{ env.NB }} + -e NBCACHE=${{ env.NBCACHE }} + -e WORKSPACE=${{ github.workspace }} + -e ALLOWERRORS=${{ env.ALLOWERRORS }} ${{ env.IMG }} - jupyter nbconvert --to notebook --execute --inplace --ExecutePreprocessor.timeout=-1 ${{ steps.jl-kernel.outputs.name }} ${{ env.NB }} + julia --project=@. ci.jl + - name: Claim output cache folder + run: | + sudo chown -R $USER ${{ env.NBCACHE }} + ls -R ${{ env.NBCACHE }} - name: Claim output notebook if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }} - run: sudo chown $USER ${{ env.NB }} - - name: Cache notebook - uses: actions/cache/save@v4 - if: ${{ steps.nb-cache.outputs.cache-hit != 'true' }} - with: - path: ${{ env.NB }} - key: ${{ steps.nb-cache.outputs.cache-primary-key }} + run: sudo chown -R $USER ${{ env.NBCACHE }} - name: Convert artifact Name id: art run: echo "name=$(echo ${{ env.NB }} | sed 's/\//-/g')" >> "$GITHUB_OUTPUT" - name: Upload Notebook uses: actions/upload-artifact@v4 with: - name: notebook-${{ needs.setup.outputs.hash }}-${{ hashFiles(env.NB) }} - path: docs/*${{ matrix.notebook }} + name: notebook-${{ steps.art.outputs.name }}-${{ needs.setup.outputs.hash }}-${{ hashFiles(env.NB) }} + path: ${{ env.NBCACHE }} include-hidden-files: true retention-days: 1 @@ -106,11 +108,11 @@ jobs: - name: Download notebooks uses: actions/download-artifact@v4 with: - path: out/ + path: ${{ env.NBCACHE }}/ pattern: notebook-* merge-multiple: true - name: Copy back built notebooks - run: cp --verbose -rf out/* docs/ + run: cp --verbose -rf ${{ env.NBCACHE }}/docs/* docs/ - name: Setup Python uses: actions/setup-python@v5 id: setup-python diff --git a/ci.jl b/ci.jl new file mode 100644 index 0000000..9e9a0c9 --- /dev/null +++ b/ci.jl @@ -0,0 +1,63 @@ +using Literate +using JSON +using Pkg +using IJulia + +ENV["GKSwstype"] = "100" + +function main(; rmsvg=true) + file = get(ENV, "NB", "test.ipynb") + cachedir = get(ENV, "NBCACHE", ".cache") + nb = if endswith(file, ".jl") + run_literate(file; cachedir) + elseif endswith(file, ".ipynb") + IJulia.installkernel("Julia", "--project=@.") + run_ipynb(file; cachedir) + else + error("$(file) is not a valid notebook file!") + end + rmsvg && strip_svg(nb) + return nothing +end + +# Strip SVG output from a Jupyter notebook +function strip_svg(ipynb) + @info "Stripping SVG in $(ipynb)" + nb = open(JSON.parse, ipynb, "r") + for cell in nb["cells"] + !haskey(cell, "outputs") && continue + for output in cell["outputs"] + !haskey(output, "data") && continue + datadict = output["data"] + if haskey(datadict, "image/png") || haskey(datadict, "image/jpeg") + delete!(datadict, "text/html") + delete!(datadict, "image/svg+xml") + end + end + end + rm(ipynb) + open(ipynb, "w") do io + JSON.print(io, nb, 1) + end + return ipynb +end + +function run_literate(file; cachedir = ".cache") + outpath = joinpath(abspath(pwd()), cachedir, dirname(file)) + mkpath(outpath) + ipynb = Literate.notebook(file, outpath; mdstrings=true, execute=true) + return ipynb +end + +function run_ipynb(file; cachedir = ".cache") + outpath = joinpath(abspath(pwd()), cachedir, file) + mkpath(dirname(outpath)) + kernelname = "--ExecutePreprocessor.kernel_name=julia-1.$(VERSION.minor)" + execute = get(ENV, "ALLOWERRORS", " ") == "true" ? "--execute --allow-errors" : "--execute" + timeout = "--ExecutePreprocessor.timeout=" * get(ENV, "TIMEOUT", "-1") + cmd = `jupyter nbconvert --to notebook $(execute) $(timeout) $(kernelname) --output $(outpath) $(file)` + run(cmd) + return outpath +end + +main() diff --git a/docs/_toc.yml b/docs/_toc.yml index 541611e..32516b7 100644 --- a/docs/_toc.yml +++ b/docs/_toc.yml @@ -5,4 +5,6 @@ format: jb-book root: index chapters: - file: pyplot + - file: pythonplot - file: sub/plots + - file: sub/plots-lit diff --git a/docs/pyplot.ipynb b/docs/pyplot.ipynb index a2b4b39..59bb01d 100644 --- a/docs/pyplot.ipynb +++ b/docs/pyplot.ipynb @@ -40,6 +40,14 @@ "plt.plot(1:5, rand(1:6, 5))" ] }, + { + "cell_type": "markdown", + "id": "e5ab6ecb", + "metadata": {}, + "source": [ + "## Runtime information" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/docs/pythonplot.jl b/docs/pythonplot.jl new file mode 100644 index 0000000..c1c9a7e --- /dev/null +++ b/docs/pythonplot.jl @@ -0,0 +1,21 @@ +#=== +# Plotting with PythonPlot.jl + +Using Literate.jl +===# +import PythonPlot as plt +using Random +Random.seed!(2022) + +#--- +plt.figure() +plt.plot(1:5, rand(1:6, 5)) +plt.gcf() + +# ## Runtime information +import Pkg +Pkg.status() + +#--- +import InteractiveUtils +InteractiveUtils.versioninfo() diff --git a/docs/sub/plots-lit.jl b/docs/sub/plots-lit.jl new file mode 100644 index 0000000..79e58b4 --- /dev/null +++ b/docs/sub/plots-lit.jl @@ -0,0 +1,19 @@ +#=== +# Plotting by Plots.jl + +Using Literate.jl +===# +using Plots +using Random +Random.seed!(2022) + +#--- +plot(rand(1:6, 5)) + +# ## Runtime information +import Pkg +Pkg.status() + +#--- +import InteractiveUtils +InteractiveUtils.versioninfo() diff --git a/docs/sub/plots.ipynb b/docs/sub/plots.ipynb index 8c5e652..3707f25 100644 --- a/docs/sub/plots.ipynb +++ b/docs/sub/plots.ipynb @@ -41,6 +41,14 @@ "plot(1:5, rand(1:6, 5))" ] }, + { + "cell_type": "markdown", + "id": "08724089", + "metadata": {}, + "source": [ + "## Runtime information" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/env.Dockerfile b/env.Dockerfile index 5c43f65..3029ef8 100644 --- a/env.Dockerfile +++ b/env.Dockerfile @@ -20,5 +20,5 @@ RUN uv pip install --system --no-cache nbconvert -r requirements.txt # Julia dependencies COPY Project.toml Manifest.toml ./ COPY src/ src -RUN julia --color=yes -e 'using Pkg; Pkg.add("IJulia"); import IJulia; IJulia.installkernel("Julia", "--project=@.")' && \ +RUN julia --color=yes -e 'using Pkg; Pkg.add(["IJulia", "Literate", "JSON"]); import IJulia; IJulia.installkernel("Julia", "--project=@.")' && \ julia --color=yes --project=@. -e 'using Pkg; Pkg.instantiate(); Pkg.precompile()'