Merge pull request #4271 from galaxyproject/alpaca-scala

Export ro-crate-metadata.json for building RO Crates of the Workflows

Gemfile

@@ -5,7 +5,7 @@
 source 'https://rubygems.org'
 gem 'addressable'
 gem 'awesome_bot'
-gem 'html-proofer'
+gem 'html-proofer', '< 5.0.0' # No specific need, it adds a lot of extra deps that we don't really need.
 gem 'jekyll'
 gem 'jekyll-feed'
 gem 'jekyll-redirect-from'
@@ -24,3 +24,6 @@ gem 'fastimage'
 
 # For our CLI tools
 gem 'commander'
+
+# RO-Crates
+gem 'rubyzip', '~> 2.3.0'

Gemfile.lock

@@ -89,16 +89,15 @@ GEM
     public_suffix (5.0.3)
     racc (1.7.1)
     rainbow (3.1.1)
-    rake (13.0.6)
     rb-fsevent (0.11.2)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
     rexml (3.2.6)
     rouge (4.1.3)
+    rubyzip (2.3.2)
     safe_yaml (1.0.5)
-    sass-embedded (1.68.0)
+    sass-embedded (1.68.0-x86_64-linux-gnu)
       google-protobuf (~> 3.23)
-      rake (>= 13.0.0)
     terminal-table (3.0.2)
       unicode-display_width (>= 1.1.1, < 3)
     typhoeus (1.4.0)
@@ -119,13 +118,14 @@ DEPENDENCIES
   commander
   csl-styles
   fastimage
-  html-proofer
+  html-proofer (< 5.0.0)
   jekyll
   jekyll-feed
   jekyll-redirect-from
   kwalify
   nokogiri (>= 1.10.4)
   pkg-config
+  rubyzip (~> 2.3.0)
   webrick
 
 BUNDLED WITH

Makefile

@@ -53,7 +53,7 @@ COND_ENV_DIR=$(shell dirname $(dir $(CONDA)))
 install: clean create-env ## install dependencies
 	$(ACTIVATE_ENV) && \
 		gem update --no-document --system && \
-		ICONV_LIBS="-L${CONDA_PREFIX}/lib/ -liconv" gem install --no-document addressable:'2.5.2' jekyll jekyll-feed jekyll-redirect-from csl-styles awesome_bot html-proofer pkg-config kwalify bibtex-ruby citeproc-ruby fastimage && \
+		ICONV_LIBS="-L${CONDA_PREFIX}/lib/ -liconv" gem install --no-document addressable:'2.5.2' jekyll jekyll-feed jekyll-redirect-from csl-styles awesome_bot html-proofer pkg-config kwalify bibtex-ruby citeproc-ruby fastimage rubyzip && \
 		pushd ${COND_ENV_DIR}/envs/${CONDA_ENV}/share/rubygems/bin && \
 		ln -sf ../../../bin/ruby ruby
 .PHONY: install

_plugins/api.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
+require 'securerandom'
 require 'json'
+require 'zip'
+
 require './_plugins/jekyll-topic-filter'
 require './_plugins/gtn/metrics'
 require './_plugins/gtn/scholar'
@@ -366,3 +369,136 @@ def generate(site)
     end
   end
 end
+
+# Basically like `PageWithoutAFile`, we just write out the ones we'd created earlier.
+Jekyll::Hooks.register :site, :post_write do |site|
+  dir = File.join(site.dest, 'api', 'workflows')
+
+  # ro-crate-metadata.json
+  TopicFilter.list_all_materials(site).select { |m| m['workflows'] }.each do |material|
+    material['workflows'].each do |workflow|
+      wfid = workflow['wfid']
+      wfname = workflow['wfname']
+      # {"workflow"=>"galaxy-workflow-mouse_novel_peptide_analysis.ga",
+      # "tests"=>false,
+      # "url"=>
+      # "http://0.0.0.0:4002/training-material/topics/.../workflows/galaxy-workflow-mouse_novel_peptide_analysis.ga",
+      # "path"=>
+      # "topics/proteomics/tutorials/.../galaxy-workflow-mouse_novel_peptide_analysis.ga",
+      # "wfid"=>"proteomics-proteogenomics-novel-peptide-analysis",
+      # "wfname"=>"galaxy-workflow-mouse_novel_peptide_analysis",
+      # "trs_endpoint"=>
+      # "http://0.0.0.0:4002/training-material/api/.../versions/galaxy-workflow-mouse_novel_peptide_analysis",
+      # "license"=>nil,
+      # "creators"=>[],
+      # "name"=>"GTN Proteogemics3 Novel Peptide Analysis",
+      # "test_results"=>nil,
+      # "modified"=>2023-06-07 12:09:36.12 +0200}
+
+      wfdir = File.join(dir, wfid, wfname)
+      FileUtils.mkdir_p(wfdir)
+      path = File.join(wfdir, 'ro-crate-metadata.json')
+      Jekyll.logger.debug "[GTN/API/WFRun] Writing #{path}"
+
+      uuids = workflow['creators'].map do |c|
+        if c.key?('identifier') && !c['identifier'].empty?
+          "https://orcid.org/#{c['identifier']}"
+        else
+          "##{SecureRandom.uuid}"
+        end
+      end
+      author_uuids = uuids.map { |u| { '@id' => u.to_s } }
+      author_linked = workflow['creators'].map.with_index do |c, i|
+        {
+          '@id' => (uuids[i]).to_s,
+          '@type' => c['class'],
+          'name' => c['name'],
+        }
+      end
+      license = workflow['license'] ? "https://spdx.org/licenses/#{workflow['license']}" : 'https://spdx.org/licenses/CC-BY-4.0'
+
+      crate = {
+        '@context' => 'https://w3id.org/ro/crate/1.1/context',
+        '@graph' => [
+          # {
+          #   '@id': './',
+          #   '@type': 'Dataset',
+          #   datePublished: workflow['modified'],
+          # },
+          {
+            '@id': 'ro-crate-metadata.json',
+            '@type': 'CreativeWork',
+            about: {
+              '@id': './'
+            },
+            conformsTo: [
+              {
+                '@id': 'https://w3id.org/ro/crate/1.1'
+              },
+              {
+                '@id': 'https://about.workflowhub.eu/Workflow-RO-Crate/'
+              }
+            ]
+          },
+          {
+            '@id': './',
+            '@type': 'Dataset',
+            datePublished: workflow['modified'].strftime('%Y-%m-%dT%H:%M:%S.%L%:z'),
+            # hasPart: [
+            #   {
+            #     '@id': '#assembly-assembly-quality-control'
+            #   }
+            # ],
+            mainEntity: {
+              '@id': "#{wfname}.ga"
+            }
+          },
+          {
+            '@id': "#{wfname}.ga",
+            '@type': %w[
+              File
+              SoftwareSourceCode
+              ComputationalWorkflow
+            ],
+            author: author_uuids,
+            license: {
+              '@id': license,
+            },
+            name: workflow['name'],
+            version: Gtn::ModificationTimes.obtain_modification_count(workflow['path']),
+            programmingLanguage: {
+              '@id': 'https://w3id.org/workflowhub/workflow-ro-crate#galaxy'
+            }
+          },
+          {
+            '@id': license,
+            '@type': 'CreativeWork',
+            name: workflow['license'],
+          },
+          {
+            '@id': 'https://w3id.org/workflowhub/workflow-ro-crate#galaxy',
+            '@type': 'ComputerLanguage',
+            identifier: {
+              '@id': 'https://galaxyproject.org/'
+            },
+            name: 'Galaxy',
+            url: {
+              '@id': 'https://galaxyproject.org/'
+            },
+            version: '23.1'
+          }
+        ]
+      }
+      crate['@graph'] += author_linked
+      File.write(path, JSON.pretty_generate(crate))
+
+      zip_path = File.join(wfdir, 'rocrate.zip')
+      Zip::File.open(zip_path, create: true) do |zipfile|
+        # - The name of the file as it will appear in the archive
+        # - The original file, including the path to find it
+        zipfile.add('ro-crate-metadata.json', path)
+        zipfile.add("#{wfname}.ga", workflow['path'])
+      end
+    end
+  end
+end

_plugins/gtn/mod.rb

@@ -6,18 +6,22 @@ module Gtn
   # This is faster than talking to the file system.
   module ModificationTimes
     @@TIME_CACHE = nil
+    @@COMMIT_COUNT_CACHE = nil
 
     def self.init_cache
       return unless @@TIME_CACHE.nil?
 
       @@TIME_CACHE = {}
+      @@COMMIT_COUNT_CACHE = Hash.new(0)
       puts '[GTN/MOD] Filling Time Cache'
-      results = `git log --name-only --pretty='GTN_GTN:%ct'`.split('GTN_GTN:')
-      results.map! { |x| x.split(/\n\n/) }
-      results.select! { |x| x.length > 1 }
-      results.each do |date, files|
+      `git log --name-only --pretty='GTN_GTN:%ct'`
+        .split('GTN_GTN:')
+        .map { |x| x.split("\n\n") }
+        .select { |x| x.length > 1 }
+        .each do |date, files|
         files.split(/\n/).each do |f|
           @@TIME_CACHE[f] = Time.at(date.to_i) if !@@TIME_CACHE.key? f
+          @@COMMIT_COUNT_CACHE[f] += 1
         end
       end
     end
@@ -26,6 +30,19 @@ def self.time_cache
       @@TIME_CACHE
     end
 
+    def self.commit_count_cache
+      @@COMMIT_COUNT_CACHE
+    end
+
+    def self.obtain_modification_count(f)
+      init_cache
+      if @@COMMIT_COUNT_CACHE.key? f
+        @@COMMIT_COUNT_CACHE[f]
+      else
+        0
+      end
+    end
+
     def self.obtain_time(f)
       init_cache
       if @@TIME_CACHE.key? f
@@ -45,5 +62,5 @@ def self.obtain_time(f)
 
 if $PROGRAM_NAME == __FILE__
   Gtn::ModificationTimes.init_cache
-  pp Gtn::ModificationTimes.time_cache
+  pp Gtn::ModificationTimes.commit_count_cache
 end

_plugins/jekyll-topic-filter.rb

@@ -486,7 +486,9 @@ def self.resolve_material(site, material)
           'trs_endpoint' => "#{domain}/#{trs}",
           'license' => license,
           'creators' => creators,
+          'name' => wf_json['name'],
           'test_results' => workflow_test_outputs,
+          'modified' => File.mtime(wf_path),
         }
       end
     end