diff --git a/Gemfile b/Gemfile index abc3eef29afddd..7161f854323a3a 100644 --- a/Gemfile +++ b/Gemfile @@ -5,7 +5,7 @@ source 'https://rubygems.org' gem 'addressable' gem 'awesome_bot' -gem 'html-proofer' +gem 'html-proofer', '< 5.0.0' # No specific need, it adds a lot of extra deps that we don't really need. gem 'jekyll' gem 'jekyll-feed' gem 'jekyll-redirect-from' @@ -24,3 +24,6 @@ gem 'fastimage' # For our CLI tools gem 'commander' + +# RO-Crates +gem 'rubyzip', '~> 2.3.0' diff --git a/Gemfile.lock b/Gemfile.lock index c7d6bf24498f29..334a3744861686 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -89,16 +89,15 @@ GEM public_suffix (5.0.3) racc (1.7.1) rainbow (3.1.1) - rake (13.0.6) rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) rexml (3.2.6) rouge (4.1.3) + rubyzip (2.3.2) safe_yaml (1.0.5) - sass-embedded (1.68.0) + sass-embedded (1.68.0-x86_64-linux-gnu) google-protobuf (~> 3.23) - rake (>= 13.0.0) terminal-table (3.0.2) unicode-display_width (>= 1.1.1, < 3) typhoeus (1.4.0) @@ -119,13 +118,14 @@ DEPENDENCIES commander csl-styles fastimage - html-proofer + html-proofer (< 5.0.0) jekyll jekyll-feed jekyll-redirect-from kwalify nokogiri (>= 1.10.4) pkg-config + rubyzip (~> 2.3.0) webrick BUNDLED WITH diff --git a/Makefile b/Makefile index 2430c43e38c304..6a62062d6e9d39 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ COND_ENV_DIR=$(shell dirname $(dir $(CONDA))) install: clean create-env ## install dependencies $(ACTIVATE_ENV) && \ gem update --no-document --system && \ - ICONV_LIBS="-L${CONDA_PREFIX}/lib/ -liconv" gem install --no-document addressable:'2.5.2' jekyll jekyll-feed jekyll-redirect-from csl-styles awesome_bot html-proofer pkg-config kwalify bibtex-ruby citeproc-ruby fastimage && \ + ICONV_LIBS="-L${CONDA_PREFIX}/lib/ -liconv" gem install --no-document addressable:'2.5.2' jekyll jekyll-feed jekyll-redirect-from csl-styles awesome_bot html-proofer pkg-config kwalify bibtex-ruby citeproc-ruby fastimage rubyzip && \ pushd ${COND_ENV_DIR}/envs/${CONDA_ENV}/share/rubygems/bin && \ ln -sf ../../../bin/ruby ruby .PHONY: install diff --git a/_plugins/api.rb b/_plugins/api.rb index 318fe610ef26c7..c76d3df1643ba6 100644 --- a/_plugins/api.rb +++ b/_plugins/api.rb @@ -1,6 +1,9 @@ # frozen_string_literal: true +require 'securerandom' require 'json' +require 'zip' + require './_plugins/jekyll-topic-filter' require './_plugins/gtn/metrics' require './_plugins/gtn/scholar' @@ -366,3 +369,136 @@ def generate(site) end end end + +# Basically like `PageWithoutAFile`, we just write out the ones we'd created earlier. +Jekyll::Hooks.register :site, :post_write do |site| + dir = File.join(site.dest, 'api', 'workflows') + + # ro-crate-metadata.json + TopicFilter.list_all_materials(site).select { |m| m['workflows'] }.each do |material| + material['workflows'].each do |workflow| + wfid = workflow['wfid'] + wfname = workflow['wfname'] + # {"workflow"=>"galaxy-workflow-mouse_novel_peptide_analysis.ga", + # "tests"=>false, + # "url"=> + # "http://0.0.0.0:4002/training-material/topics/.../workflows/galaxy-workflow-mouse_novel_peptide_analysis.ga", + # "path"=> + # "topics/proteomics/tutorials/.../galaxy-workflow-mouse_novel_peptide_analysis.ga", + # "wfid"=>"proteomics-proteogenomics-novel-peptide-analysis", + # "wfname"=>"galaxy-workflow-mouse_novel_peptide_analysis", + # "trs_endpoint"=> + # "http://0.0.0.0:4002/training-material/api/.../versions/galaxy-workflow-mouse_novel_peptide_analysis", + # "license"=>nil, + # "creators"=>[], + # "name"=>"GTN Proteogemics3 Novel Peptide Analysis", + # "test_results"=>nil, + # "modified"=>2023-06-07 12:09:36.12 +0200} + + wfdir = File.join(dir, wfid, wfname) + FileUtils.mkdir_p(wfdir) + path = File.join(wfdir, 'ro-crate-metadata.json') + Jekyll.logger.debug "[GTN/API/WFRun] Writing #{path}" + + uuids = workflow['creators'].map do |c| + if c.key?('identifier') && !c['identifier'].empty? + "https://orcid.org/#{c['identifier']}" + else + "##{SecureRandom.uuid}" + end + end + author_uuids = uuids.map { |u| { '@id' => u.to_s } } + author_linked = workflow['creators'].map.with_index do |c, i| + { + '@id' => (uuids[i]).to_s, + '@type' => c['class'], + 'name' => c['name'], + } + end + license = workflow['license'] ? "https://spdx.org/licenses/#{workflow['license']}" : 'https://spdx.org/licenses/CC-BY-4.0' + + crate = { + '@context' => 'https://w3id.org/ro/crate/1.1/context', + '@graph' => [ + # { + # '@id': './', + # '@type': 'Dataset', + # datePublished: workflow['modified'], + # }, + { + '@id': 'ro-crate-metadata.json', + '@type': 'CreativeWork', + about: { + '@id': './' + }, + conformsTo: [ + { + '@id': 'https://w3id.org/ro/crate/1.1' + }, + { + '@id': 'https://about.workflowhub.eu/Workflow-RO-Crate/' + } + ] + }, + { + '@id': './', + '@type': 'Dataset', + datePublished: workflow['modified'].strftime('%Y-%m-%dT%H:%M:%S.%L%:z'), + # hasPart: [ + # { + # '@id': '#assembly-assembly-quality-control' + # } + # ], + mainEntity: { + '@id': "#{wfname}.ga" + } + }, + { + '@id': "#{wfname}.ga", + '@type': %w[ + File + SoftwareSourceCode + ComputationalWorkflow + ], + author: author_uuids, + license: { + '@id': license, + }, + name: workflow['name'], + version: Gtn::ModificationTimes.obtain_modification_count(workflow['path']), + programmingLanguage: { + '@id': 'https://w3id.org/workflowhub/workflow-ro-crate#galaxy' + } + }, + { + '@id': license, + '@type': 'CreativeWork', + name: workflow['license'], + }, + { + '@id': 'https://w3id.org/workflowhub/workflow-ro-crate#galaxy', + '@type': 'ComputerLanguage', + identifier: { + '@id': 'https://galaxyproject.org/' + }, + name: 'Galaxy', + url: { + '@id': 'https://galaxyproject.org/' + }, + version: '23.1' + } + ] + } + crate['@graph'] += author_linked + File.write(path, JSON.pretty_generate(crate)) + + zip_path = File.join(wfdir, 'rocrate.zip') + Zip::File.open(zip_path, create: true) do |zipfile| + # - The name of the file as it will appear in the archive + # - The original file, including the path to find it + zipfile.add('ro-crate-metadata.json', path) + zipfile.add("#{wfname}.ga", workflow['path']) + end + end + end +end diff --git a/_plugins/gtn/mod.rb b/_plugins/gtn/mod.rb index 12cb9a8378a382..cc6575f617f367 100644 --- a/_plugins/gtn/mod.rb +++ b/_plugins/gtn/mod.rb @@ -6,18 +6,22 @@ module Gtn # This is faster than talking to the file system. module ModificationTimes @@TIME_CACHE = nil + @@COMMIT_COUNT_CACHE = nil def self.init_cache return unless @@TIME_CACHE.nil? @@TIME_CACHE = {} + @@COMMIT_COUNT_CACHE = Hash.new(0) puts '[GTN/MOD] Filling Time Cache' - results = `git log --name-only --pretty='GTN_GTN:%ct'`.split('GTN_GTN:') - results.map! { |x| x.split(/\n\n/) } - results.select! { |x| x.length > 1 } - results.each do |date, files| + `git log --name-only --pretty='GTN_GTN:%ct'` + .split('GTN_GTN:') + .map { |x| x.split("\n\n") } + .select { |x| x.length > 1 } + .each do |date, files| files.split(/\n/).each do |f| @@TIME_CACHE[f] = Time.at(date.to_i) if !@@TIME_CACHE.key? f + @@COMMIT_COUNT_CACHE[f] += 1 end end end @@ -26,6 +30,19 @@ def self.time_cache @@TIME_CACHE end + def self.commit_count_cache + @@COMMIT_COUNT_CACHE + end + + def self.obtain_modification_count(f) + init_cache + if @@COMMIT_COUNT_CACHE.key? f + @@COMMIT_COUNT_CACHE[f] + else + 0 + end + end + def self.obtain_time(f) init_cache if @@TIME_CACHE.key? f @@ -45,5 +62,5 @@ def self.obtain_time(f) if $PROGRAM_NAME == __FILE__ Gtn::ModificationTimes.init_cache - pp Gtn::ModificationTimes.time_cache + pp Gtn::ModificationTimes.commit_count_cache end diff --git a/_plugins/jekyll-topic-filter.rb b/_plugins/jekyll-topic-filter.rb index ad638b012357e9..de25f52e872bdb 100644 --- a/_plugins/jekyll-topic-filter.rb +++ b/_plugins/jekyll-topic-filter.rb @@ -486,7 +486,9 @@ def self.resolve_material(site, material) 'trs_endpoint' => "#{domain}/#{trs}", 'license' => license, 'creators' => creators, + 'name' => wf_json['name'], 'test_results' => workflow_test_outputs, + 'modified' => File.mtime(wf_path), } end end