Update docker file for building site

.github/workflows/build.yml

@@ -14,6 +14,9 @@ on:
 
 env:
   PARTICIPANT_DATA:
+  REGISTRY: ghcr.io
+  ORG: nbisweden
+  IMAGE: workshop-pgip/workshop-pgip:latest
 
 jobs:
   build:
@@ -26,6 +29,13 @@ jobs:
       run:
         shell: bash -l {0}
     steps:
+      - name: Set time zone
+        uses: szenius/[email protected]
+        with:
+          timezoneLinux: "Europe/Stockholm"
+          timezoneMacos: "Europe/Stockholm"
+          timezoneWindows: "W. Europe Standard Time"
+
       - name: Cancel Previous Runs
         if: ${{ !env.ACT }}
         uses: styfle/[email protected]
@@ -35,24 +45,26 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Install micromamba
-        uses: mamba-org/setup-micromamba@v1
-        with:
-          micromamba-version: "1.3.1-0"
-          environment-file: conda-linux-64.lock
-          environment-name: pgip
-          init-shell: bash
-          cache-environment: true
-
-      - name: Test git installation
-        run: |
-          git --help
-          echo Listing branches
-          git branch -l
-          echo Fetching branches
-          git fetch --all
-          echo Listing branches again
-          git branch --list
+      - name: Pull image
+        run: docker pull ${{ env.REGISTRY }}/${{ env.ORG }}/${{ env.IMAGE }}
+      # - name: Install micromamba
+      #   uses: mamba-org/setup-micromamba@v1
+      #   with:
+      #     micromamba-version: "1.3.1-0"
+      #     environment-file: conda-linux-64.lock
+      #     environment-name: pgip
+      #     init-shell: bash
+      #     cache-environment: true
+
+      # - name: Test git installation
+      #   run: |
+      #     git --help
+      #     echo Listing branches
+      #     git branch -l
+      #     echo Fetching branches
+      #     git fetch --all
+      #     echo Listing branches again
+      #     git branch --list
 
       - name: Install pgip dependencies
         run: |
@@ -121,13 +133,27 @@ jobs:
           fi
 
       - name: Render tag and save rendered state in separate branch
-        uses: quarto-dev/quarto-actions/publish@v2
         if: startsWith(github.ref, 'refs/tags/v')
-        with:
-          target: "archive-${{ github.ref_name }}"
-          path: docs
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_EMAIL: [email protected]
+          GITHUB_USERNAME: gh_action
+          TARGET: "archive-${{ github.ref_name }}"
+        run: |
+          # Add git info
+          git config --global user.email "${{ env.GITHUB_EMAIL }}"
+          git config --global user.name "${{ env.GITHUB_USERNAME }}"
+          git clone --single-branch --branch ${{ env.TARGET }} "https://${{ secrets.GITHUB_TOKEN }}@github.com/${GITHUB_REPOSITORY}.git" tmprepo
+
+
+
+      # - name: Render tag and save rendered state in separate branch
+      #   uses: quarto-dev/quarto-actions/publish@v2
+      #   if: startsWith(github.ref, 'refs/tags/v')
+      #   with:
+      #     target: "archive-${{ github.ref_name }}"
+      #     path: docs
+      #   env:
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Checkout archived course versions
         env:

.pre-commit-config.yaml

@@ -108,3 +108,16 @@ repos:
         files: '(\.[rR]profile|\.R|\.Rmd|\.Rnw|\.r|\.rmd|\.rnw)$'
         exclude: 'renv/activate\.R'
         minimum_pre_commit_version: "2.13.0"
+  - repo: local
+    hooks:
+      - id: pylint
+        name: pylint
+        entry: pylint
+        language: system
+        types: [python]
+        args:
+          [
+          "-rn", # Only display messages
+          "-sn", # Don't display the score
+          "--disable=fixme",
+          ]

docs/_quarto.yml

@@ -5,7 +5,7 @@ project:
   pre-render:
     - ../scripts/init-site.sh
     - ../scripts/setup-pgip-data.sh
-    - ../scripts/setup-data.py
+    - ../scripts/setup_data.py
   post-render:
     - ../scripts/post-render.py
   render:

environment.yml

@@ -12,13 +12,14 @@ dependencies:
   - bwa
   - ca-certificates
   - csvtk
-  - cyvcf2=0.30.16
+  - cyvcf2
   - demesdraw
   - d4tools
   - fastqc
   - gatk4
   - gawk
   - htslib=1.15.1
+  - ipykernel
   - jupyter
   - jupyter-cache
   - mosdepth=0.3.3

scripts/Dockerfile

@@ -0,0 +1,55 @@
+# Docker file for workshop-pgip
+# 2023 Per Unneberg
+
+FROM rocker/verse:4.2.3
+LABEL Description="Docker image for NBIS workshop Population Genomics in Practice"
+LABEL Maintainer="[email protected]"
+LABEL org.opencontainers.image.source="https://github.com/NBISweden/workshop-pgip"
+ARG QUARTO_VERSION="1.3.450"
+ENV PATH="${PATH}:/miniconda3/bin"
+
+RUN apt-get update -y \
+    && apt-get install --no-install-recommends -y \
+    libxml2-dev \
+    libssl-dev \
+    libcurl4-openssl-dev \
+    libudunits2-dev \
+    libgdal-dev \
+    texlive \
+    texlive-latex-extra \
+    && wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb \
+    && apt-get install -y ./google-chrome-stable_current_amd64.deb \
+    && rm -rf ./google-chrome-stable_current_amd64.deb \
+    && curl -o quarto-linux-amd64.deb -L https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.deb \
+    && apt-get install -y ./quarto-linux-amd64.deb \
+    && rm -rf ./quarto-linux-amd64.deb \
+    && Rscript -e 'install.packages(c("ape", "codetools", "curl", "devtools", "dt", "dygraphs", "expm", "fontawesome", "formatr"), repos = "http://cran.us.r-project.org");' \
+    && Rscript -e 'install.packages(c("formattable", "gganimate", "ggiraph", "ggraph", "ggplot2", "gifski", "gplots"), repos = "http://cran.us.r-project.org");' \
+    && Rscript -e 'install.packages(c("gsalib", "highcharter", "igraph", "kableExtra", "latex2exp", "leaflet"), repos = "http://cran.us.r-project.org");' \
+    && Rscript -e 'install.packages(c("networkD3", "magick", "patchwork", "pdftools", "plotly", "readr", "readxl", "reticulate"), repos = "http://cran.us.r-project.org");' \
+    && Rscript -e 'install.packages(c("rmarkdown", "tidyr", "viridis"), repos = "http://cran.us.r-project.org");' \
+    && rm -rf /var/lib/apt/lists/* \
+    && mkdir /qmd
+
+RUN mkdir -p /miniconda3 \
+    && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /miniconda3/miniconda.sh \
+    && bash /miniconda3/miniconda.sh -b -u -p /miniconda3 \
+    && rm -rf /miniconda3/miniconda.sh
+
+RUN conda init bash
+
+COPY env.yaml /tmp/env.yaml
+RUN conda env update -n base -f /tmp/env.yaml
+ENV PATH /miniconda3/envs/base/bin:$PATH
+RUN /bin/bash -c "source activate base"
+
+RUN python -m ipykernel install --user --display-name "Population Genomics in Practice (Python)"
+RUN python -m pip install git+https://github.com/NBISweden/pgip-tools
+
+RUN Rscript -e 'install.packages("dotenv", repos=c(CRAN = "https://cran.rstudio.com/"));' \
+    R -e "library(devtools); devtools::install_local('../src/R/pgip', upgrade=FALSE)"
+
+WORKDIR /qmd
+ENV XDG_CACHE_HOME=/tmp/quarto_cache_home
+ENV XDG_DATA_HOME=/tmp/quarto_data_home
+CMD quarto render

scripts/env.yaml

@@ -0,0 +1,44 @@
+## Environment file to reproduce exercises
+name: base
+channels:
+  - conda-forge
+  - bioconda
+  - defaults
+dependencies:
+  - bash_kernel
+  - bedtools
+  - bcftools
+  - bokeh
+  - bwa
+  - ca-certificates
+  - csvtk
+#  - cyvcf2
+  - demesdraw
+  - d4tools
+  - fastqc
+  - gatk4
+  - gawk
+#  - htslib=1.15.1
+  - ipykernel
+  - jupyter
+  - jupyter-cache
+  - mosdepth # =0.3.3
+  - msprime>=1.2
+  - multiqc
+  - pandas
+  - pip
+  - picard #<3
+  - pyslim
+  - python-newick
+  - qualimap
+  - samtools=1.15.1
+  - scikit-allel
+  - seqkit
+  - slim>=4.0
+  - tskit>=0.5.0
+  - vcflib=1.0.1
+  - vcftools
+  - pip:
+    - cyvcf2
+    - pixy
+    - sgkit[vcf]

scripts/post-render.py → scripts/post_render.py

@@ -1,30 +1,33 @@
 #!/usr/bin/env python
+"""Perform post rendering tasks"""
 import errno
 import logging
 import os
 import shutil
 
 
-def safe_mkdir(dirname):
-    logging.debug(f"Making directory {dirname}")
-    if not os.path.exists(dirname):
+def safe_mkdir(dname):
+    """Make directory if it doesn't exist"""
+    logging.debug("Making directory %s", dname)
+    if not os.path.exists(dname):
         try:
-            os.mkdir(dirname)
-        except FileNotFoundError as e:
-            print(e)
+            os.mkdir(dname)
+        except FileNotFoundError as exc:
+            print(exc)
             raise
 
 
-def safe_copy(src, dst):
-    logging.debug(f"Copy {dst} -> {src}")
-    if not os.path.exists(src):
-        raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), src)
+def safe_copy(source, destination):
+    """Copy source to destination"""
+    logging.debug("Copy %s -> %s", destination, source)
+    if not os.path.exists(source):
+        raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), source)
     try:
-        shutil.copy(src, dst)
+        shutil.copy(source, destination)
     except FileExistsError:
-        logging.debug(f"Path {dst} exists; skipping")
-    except Exception as e:
-        print(e)
+        logging.debug("Path %s exists; skipping", destination)
+    except Exception as exc:
+        print(exc)
         raise
 
 

scripts/setup-data.py → scripts/setup_data.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+"""Setup data examples for rendering exercises and lectures"""
 import argparse
 import errno
 import logging
@@ -7,28 +8,31 @@
 import yaml
 
 ROOT = os.path.normpath(os.path.join(os.path.dirname(__file__), os.pardir))
-
-DATAROOT = os.environ.get("DATA_SOURCE", "pgip_data/data")
+DOCROOT = os.path.join(ROOT, "docs")
+DATAROOT = os.path.join(DOCROOT, "pgip_data/data")
 
 
 def abspath(path):
+    """Return absolute path"""
     return os.path.normpath(os.path.abspath(path))
 
 
 def safe_link(src, dst, dir_fd):
-    logging.debug(f"Link {dst} -> {src}")
+    """Create a symlink from src to dst in directory dir_fd"""
+    logging.debug("Link %s -> %s", dst, src)
     if not os.path.exists(src):
         raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), src)
     try:
         os.symlink(src, dst, dir_fd=dir_fd)
     except FileExistsError:
-        logging.debug(f"Path {dst} exists; skipping")
-    except Exception as e:
-        print(e)
+        logging.debug("Path %s exists; skipping", dst)
+    except Exception as exc:
+        print(exc)
         raise
 
 
 def main():
+    """Setup data examples for rendering exercises and lectures"""
     parser = argparse.ArgumentParser(
         prog="setup-data.py",
         description="Setup data examples for rendering exercises and lectures",
@@ -41,15 +45,15 @@ def main():
 
     logging.basicConfig(level=args.loglevel.upper())
 
-    with open(args.config, "r") as fh:
+    with open(args.config, "r", encoding="utf-8") as fh:
         pgip_data = yaml.safe_load(fh)
 
     # Loop through data and link files
     for session, dataset in pgip_data.items():
         if session.startswith("__"):
             continue
-        logging.info(f"Setting up links for {session}")
-        data = dict()
+        logging.info("Setting up links for %s", session)
+        data = {}
         for x in dataset:
             if isinstance(x, dict):
                 data.update(**x)