From 45d9816a846f4579f5dce25d34add0fec980ad7e Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 4 Dec 2023 15:22:34 -0600 Subject: [PATCH 01/20] Mirador: only load on item view. --- app/helpers/application_helper.rb | 15 +++++++++++++++ app/views/layouts/blacklight/base.html.erb | 6 +++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index ad38607b6..a8a84ae9f 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -139,4 +139,19 @@ def score_output(args) def render_sidebar_map?(document) Settings.SIDEBAR_STATIC_MAP&.any? { |vp| document.viewer_protocol == vp } end + + ## + # Gets current layout for use in rendering partials + # @return [String] item, index, home, or default + def layout_type + if params[:controller] == "catalog" + if params[:action] == "show" || params[:action] == "downloads" + "item" + elsif params[:action] == "index" + "index" + end + else + "default" + end + end end diff --git a/app/views/layouts/blacklight/base.html.erb b/app/views/layouts/blacklight/base.html.erb index 99b570196..0af7afb2c 100644 --- a/app/views/layouts/blacklight/base.html.erb +++ b/app/views/layouts/blacklight/base.html.erb @@ -13,7 +13,11 @@ <%= opensearch_description_tag application_name, opensearch_catalog_url(format: 'xml') %> <%= favicon_link_tag %> <%= stylesheet_link_tag "application", media: "all" %> - <%= javascript_pack_tag 'm3' %> + + <% if layout_type == "item" %> + <%= javascript_pack_tag 'm3' %> + <% end %> + <%= javascript_include_tag "application" %> <%= javascript_include_tag "analytics" %> From 9f82b291031d11e34ba6106691fe00bd86905eba Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 4 Dec 2023 15:44:52 -0600 Subject: [PATCH 02/20] Layout: only load m3 (mirador) on iiif_manifest viewer_protocol objects. Fixes #548 --- app/views/layouts/blacklight/base.html.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/layouts/blacklight/base.html.erb b/app/views/layouts/blacklight/base.html.erb index 0af7afb2c..c133e7e15 100644 --- a/app/views/layouts/blacklight/base.html.erb +++ b/app/views/layouts/blacklight/base.html.erb @@ -14,10 +14,10 @@ <%= favicon_link_tag %> <%= stylesheet_link_tag "application", media: "all" %> - <% if layout_type == "item" %> + <% if layout_type == "item" && @document && @document.viewer_protocol == 'iiif_manifest' %> <%= javascript_pack_tag 'm3' %> <% end %> - + <%= javascript_include_tag "application" %> <%= javascript_include_tag "analytics" %> From 2f5e6acb89f46cc8dbaa220e55e448cabbee415b Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Wed, 6 Dec 2023 11:27:08 -0600 Subject: [PATCH 03/20] Tableau Export: stream CSV from database Will provide the data necessary for tableau viz much faster than the GBL Admin CSV export. --- app/controllers/admin/api_controller.rb | 42 +++++++++++++++++++++++++ app/views/robots/robots.text.erb | 1 + config/routes.rb | 1 + 3 files changed, 44 insertions(+) diff --git a/app/controllers/admin/api_controller.rb b/app/controllers/admin/api_controller.rb index eb4b66867..3646ef7b2 100644 --- a/app/controllers/admin/api_controller.rb +++ b/app/controllers/admin/api_controller.rb @@ -338,6 +338,48 @@ class ApiController < ApplicationController config.autocomplete_path = 'suggest' end + # Admin CSV stream of Database data + def tableau_export + headers.delete("Content-Length") + headers["Cache-Control"] = "no-cache" + headers["Content-Type"] = "text/csv" + headers["Content-Disposition"] = "attachment; filename=\"geoportal_tableau_export.csv\"" + headers["X-Accel-Buffering"] = "no" + response.status = 200 + self.response_body = csv_enumerator + end + + def csv_enumerator + @csv_enumerator ||= Enumerator.new do |yielder| + yielder << CSV.generate_line([ + "Title", + "Provider", + "Resource Class", + "Resource Type", + "Index Year", + "Spatial Coverage", + "B1G Image", + "ID", + "Download", + "Language" + ]) + Document.find_each do |row| + yielder << CSV.generate_line([ + row.title, + row.schema_provider_s, + row.gbl_resourceClass_sm&.join("|"), + row.gbl_resourceType_sm&.join("|"), + row.gbl_indexYear_im&.join("|"), + row.dct_spatial_sm&.join("|"), + row.b1g_image_ss, + row.geomg_id_s, + row.dct_references_s.find { |ref| ref.category == "download" }&.value, + row.dct_language_sm&.join("|") + ]) + end + end + end + # Administrative view of document # - Sidecar Image # - URIs diff --git a/app/views/robots/robots.text.erb b/app/views/robots/robots.text.erb index b44d31232..62df1dc03 100644 --- a/app/views/robots/robots.text.erb +++ b/app/views/robots/robots.text.erb @@ -14,6 +14,7 @@ Disallow: /?f Disallow: /?_ Disallow: /?bbox Disallow: /?page= +Disallow: /admin/api/tableau_export Disallow: /catalog.html?f Disallow: /catalog.html?_ Disallow: /catalog.atom diff --git a/config/routes.rb b/config/routes.rb index 2aea4eb29..fc90c4727 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -158,6 +158,7 @@ get '/api' => 'api#index', constraints: lambda { |req| req.format == :json } get '/api/fetch' => 'api#fetch', constraints: lambda { |req| req.format == :json } get '/api/facet/:id' => 'api#facet', constraints: lambda { |req| req.format == :json } + get '/api/tableau_export' => 'api#tableau_export' # Documents resources :documents do From 835325922986d2674900c241c74602cd1548ad03 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Thu, 7 Dec 2023 13:21:38 -0600 Subject: [PATCH 04/20] Admin::API - Add Date Accessioned to advanced search page --- app/controllers/admin/api_controller.rb | 3 +++ config/settings.yml | 1 + 2 files changed, 4 insertions(+) diff --git a/app/controllers/admin/api_controller.rb b/app/controllers/admin/api_controller.rb index 3646ef7b2..a799047fe 100644 --- a/app/controllers/admin/api_controller.rb +++ b/app/controllers/admin/api_controller.rb @@ -141,6 +141,9 @@ class ApiController < ApplicationController # ADVANCED SEARCH # + # Accession Date + config.add_facet_field Settings.FIELDS.B1G_DATE_ACCESSIONED, :label => 'Date Accessioned', :limit => 8, collapse: false, sort: "index" + # Code config.add_facet_field Settings.FIELDS.B1G_CODE, label: 'Code', show: false diff --git a/config/settings.yml b/config/settings.yml index a743b85c9..4414fedd3 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -92,6 +92,7 @@ FIELDS: :WXS_IDENTIFIER: 'gbl_wxsIdentifier_s' # B1G Custom Fields Geoportal / GEOMG :B1G_ACCESS: 'b1g_access_s' + :B1G_DATE_ACCESSIONED: 'b1g_dateAccessioned_sm' :B1G_ACCRUAL_METHOD: 'b1g_dct_accrualMethod_s' :B1G_CHILD_RECORD: 'b1g_child_record_b' :B1G_CODE: 'b1g_code_s' From efeca2d2a99971dbd094d3a9d0486acdd6bf2595 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Thu, 7 Dec 2023 14:24:29 -0600 Subject: [PATCH 05/20] GBL Admin: latest work --- Gemfile | 2 +- Gemfile.lock | 76 ++++++++++--------- .../admin/advanced_search_controller.rb | 19 ++++- app/controllers/admin/api_controller.rb | 3 + app/controllers/catalog_controller.rb | 4 +- app/models/search_builder.rb | 1 + config/settings.yml | 1 + 7 files changed, 67 insertions(+), 39 deletions(-) diff --git a/Gemfile b/Gemfile index 1c2086099..7b07890b3 100644 --- a/Gemfile +++ b/Gemfile @@ -80,7 +80,7 @@ gem 'bootstrap', '~> 4.0' gem 'popper_js' gem 'twitter-typeahead-rails', '0.11.1.pre.corejavascript' gem 'geoblacklight', '~> 4.1' -gem 'geoblacklight_admin', '~> 0.3' +gem 'geoblacklight_admin', git: "https://github.com/geobtaa/geoblacklight_admin.git", branch: "feature/bulk-actions-sti" gem "rubyzip", ">= 1.3.0" gem "awesome_print" diff --git a/Gemfile.lock b/Gemfile.lock index 1b3e50fad..240954114 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -39,6 +39,42 @@ GIT rails (>= 5.2, < 7.1) statesman (~> 10.0.0) +GIT + remote: https://github.com/geobtaa/geoblacklight_admin.git + revision: d139c5e92480a25c28212dabd5233c7165522dfa + branch: feature/bulk-actions-sti + specs: + geoblacklight_admin (0.3.2) + active_storage_validations (~> 1.0) + amazing_print + blacklight (~> 7.33) + blacklight_advanced_search + blacklight_range_limit + bootstrap (~> 4.0) + chosen-rails (~> 1.10) + cocoon (~> 1.2) + config (~> 4.0) + devise (~> 4.7) + devise-bootstrap-views (~> 1.0) + devise_invitable (~> 2.0) + dotenv-rails (~> 2.8) + geoblacklight (~> 4.0) + haml (~> 5.2) + httparty (~> 0.21) + inline_svg (~> 1.9) + jquery-rails (~> 4.4) + kithe (~> 2.0) + noticed (~> 1.6) + pagy (~> 6.0) + paper_trail (~> 14.0) + pg (~> 1.4) + qa (~> 5.0) + rails (>= 6.1, < 7.1) + ruby-progressbar + simple_form (~> 5.0) + sprockets (< 4) + statesman (~> 10.0) + GEM remote: https://rubygems.org/ specs: @@ -96,7 +132,7 @@ GEM activerecord (6.1.7.6) activemodel (= 6.1.7.6) activesupport (= 6.1.7.6) - activerecord-import (1.5.0) + activerecord-import (1.5.1) activerecord (>= 4.2) activestorage (6.1.7.6) actionpack (= 6.1.7.6) @@ -328,36 +364,6 @@ GEM mime-types rails (>= 6.1, < 7.1) rgeo-geojson - geoblacklight_admin (0.3.2) - active_storage_validations (~> 1.0) - amazing_print - blacklight (~> 7.33) - blacklight_advanced_search - blacklight_range_limit - bootstrap (~> 4.0) - chosen-rails (~> 1.10) - cocoon (~> 1.2) - config (~> 4.0) - devise (~> 4.7) - devise-bootstrap-views (~> 1.0) - devise_invitable (~> 2.0) - dotenv-rails (~> 2.8) - geoblacklight (~> 4.0) - haml (~> 5.2) - httparty (~> 0.21) - inline_svg (~> 1.9) - jquery-rails (~> 4.4) - kithe (~> 2.0) - noticed (~> 1.6) - pagy (~> 6.0) - paper_trail (~> 14.0) - pg (~> 1.4) - qa (~> 5.0) - rails (>= 6.1, < 7.1) - ruby-progressbar - simple_form (~> 5.0) - sprockets (< 4) - statesman (~> 10.0) geocoder (1.8.2) git (1.11.0) rchardet (~> 1.8) @@ -507,7 +513,7 @@ GEM rails (>= 5.2.0) orm_adapter (0.5.0) ostruct (0.5.5) - pagy (6.1.0) + pagy (6.2.0) paper_trail (14.0.0) activerecord (>= 6.0) request_store (~> 1.4) @@ -530,14 +536,14 @@ GEM public_suffix (3.1.1) puma (5.6.7) nio4r (~> 2.0) - qa (5.10.0) + qa (5.11.0) activerecord-import deprecation faraday (< 3.0, != 2.0.0) geocoder ldpath nokogiri (~> 1.6) - rails (>= 5.0, < 7.1) + rails (>= 5.0, < 7.2) rdf racc (1.7.1) rack (2.2.8) @@ -791,7 +797,7 @@ DEPENDENCIES foreman geoblacklight (~> 4.1) geoblacklight-icons! - geoblacklight_admin (~> 0.3) + geoblacklight_admin! geoblacklight_sidecar_images! haml handlebars_assets (~> 0.23.0) diff --git a/app/controllers/admin/advanced_search_controller.rb b/app/controllers/admin/advanced_search_controller.rb index 41a737c55..36328e219 100644 --- a/app/controllers/admin/advanced_search_controller.rb +++ b/app/controllers/admin/advanced_search_controller.rb @@ -33,7 +33,21 @@ class AdvancedSearchController < ApplicationController config.advanced_search[:url_key] ||= 'advanced' config.advanced_search[:query_parser] ||= 'edismax' config.advanced_search[:form_solr_parameters] ||= {} - config.advanced_search[:form_solr_parameters]['facet.field'] ||= [Settings.FIELDS.PROVIDER, Settings.FIELDS.B1G_CODE, Settings.FIELDS.MEMBER_OF, Settings.FIELDS.IS_PART_OF, Settings.FIELDS.RESOURCE_CLASS, Settings.FIELDS.RESOURCE_TYPE, Settings.FIELDS.SUBJECT, Settings.FIELDS.THEME, Settings.FIELDS.FORMAT, Settings.FIELDS.SUPPRESSED, Settings.FIELDS.B1G_CHILD_RECORD, Settings.FIELDS.GEOREFERENCED] + config.advanced_search[:form_solr_parameters]['facet.field'] ||= [ + Settings.FIELDS.PROVIDER, + Settings.FIELDS.B1G_DATE_ACCESSIONED, + Settings.FIELDS.B1G_CODE, + Settings.FIELDS.MEMBER_OF, + Settings.FIELDS.IS_PART_OF, + Settings.FIELDS.RESOURCE_CLASS, + Settings.FIELDS.RESOURCE_TYPE, + Settings.FIELDS.SUBJECT, + Settings.FIELDS.THEME, + Settings.FIELDS.FORMAT, + Settings.FIELDS.SUPPRESSED, + Settings.FIELDS.B1G_CHILD_RECORD, + Settings.FIELDS.GEOREFERENCED + ] config.advanced_search[:form_solr_parameters]['facet.query'] ||= '' config.advanced_search[:form_solr_parameters]['facet.limit'] ||= -1 config.advanced_search[:form_solr_parameters]['facet.sort'] ||= 'index' @@ -128,6 +142,9 @@ class AdvancedSearchController < ApplicationController # ADVANCED SEARCH # + # B1G Date Accessioned + config.add_facet_field Settings.FIELDS.B1G_DATE_ACCESSIONED, label: 'Date Accessioned', limit: 1000 + # Code config.add_facet_field Settings.FIELDS.B1G_CODE, label: 'Code', limit: 1000 diff --git a/app/controllers/admin/api_controller.rb b/app/controllers/admin/api_controller.rb index a799047fe..7f78e005f 100644 --- a/app/controllers/admin/api_controller.rb +++ b/app/controllers/admin/api_controller.rb @@ -127,6 +127,9 @@ class ApiController < ApplicationController # Publication State config.add_facet_field Settings.FIELDS.B1G_PUBLICATION_STATE, :label => 'Publication State', :limit => 8, collapse: false + # Import ID + config.add_facet_field Settings.FIELDS.B1G_IMPORT_ID, label: "Import ID", show: false + # Resouce Class config.add_facet_field Settings.FIELDS.RESOURCE_CLASS, label: 'Resource Class', limit: 8 diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb index 0ae0dddfd..899a8c5a4 100644 --- a/app/controllers/catalog_controller.rb +++ b/app/controllers/catalog_controller.rb @@ -58,8 +58,7 @@ class CatalogController < ApplicationController :start => 0, 'q.alt' => '*:*', 'fl' => '*,score,[explain]', - 'bf' => ["if(exists(#{Settings.FIELDS.B1G_CHILD_RECORD}),0,100)^0.5"], - 'fq' => ["#{Settings.FIELDS.B1G_PUBLICATION_STATE}:published"] + 'bf' => ["if(exists(#{Settings.FIELDS.B1G_CHILD_RECORD}),0,100)^0.5"] } config.default_per_page = 10 # Works! @@ -157,6 +156,7 @@ class CatalogController < ApplicationController # filter_class - Defines how to add/remove facet from query # label - Defines the label used in contstraints container config.add_facet_field 'solr_bboxtype', item_presenter: Geoblacklight::BboxItemPresenter, filter_class: Geoblacklight::BboxFilterField, filter_query_builder: Geoblacklight::BboxFilterQuery, within_boost: Settings.BBOX_WITHIN_BOOST, overlap_boost: Settings.OVERLAP_RATIO_BOOST, overlap_field: Settings.FIELDS.OVERLAP_FIELD, label: 'Bounding Box' + config.add_facet_field Settings.FIELDS.B1G_IMPORT_ID, label: "Import ID", show: false # Item Relationship Facets # * Not displayed to end user (show: false) diff --git a/app/models/search_builder.rb b/app/models/search_builder.rb index 369e82a87..8f9df0037 100644 --- a/app/models/search_builder.rb +++ b/app/models/search_builder.rb @@ -2,6 +2,7 @@ class SearchBuilder < Blacklight::SearchBuilder include Blacklight::Solr::SearchBuilderBehavior include BlacklightAdvancedSearch::AdvancedSearchBuilder include B1gDateRangeQueryConcern::RangeLimitBuilder + include GeoblacklightAdmin::PublicationStateSearchBehavior self.default_processor_chain += [:add_advanced_parse_q_to_solr, :add_advanced_search_to_solr] diff --git a/config/settings.yml b/config/settings.yml index 4414fedd3..70ad56128 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -102,6 +102,7 @@ FIELDS: :B1G_LANGUAGE: 'b1g_language_sm' :B1G_MEDIATOR: 'b1g_dct_mediator_sm' :B1G_PUBLICATION_STATE: 'b1g_publication_state_s' + :B1G_IMPORT_ID: 'b1g_geom_import_id_ssi' GEOMONITOR_TOLERANCE: 0.8 #GOOGLE_ANALYTICS_ID: 'UA-XXXXX-X' From 3215e92d44c45b0f1cac7cc7e3f9192b132f800e Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Thu, 7 Dec 2023 14:34:45 -0600 Subject: [PATCH 06/20] Update api_controller.rb Move Accession Date to preferred location. --- app/controllers/admin/api_controller.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/controllers/admin/api_controller.rb b/app/controllers/admin/api_controller.rb index 7f78e005f..a1021c316 100644 --- a/app/controllers/admin/api_controller.rb +++ b/app/controllers/admin/api_controller.rb @@ -127,6 +127,9 @@ class ApiController < ApplicationController # Publication State config.add_facet_field Settings.FIELDS.B1G_PUBLICATION_STATE, :label => 'Publication State', :limit => 8, collapse: false + # Accession Date + config.add_facet_field Settings.FIELDS.B1G_DATE_ACCESSIONED, :label => 'Date Accessioned', :limit => 8, collapse: false + # Import ID config.add_facet_field Settings.FIELDS.B1G_IMPORT_ID, label: "Import ID", show: false @@ -144,8 +147,6 @@ class ApiController < ApplicationController # ADVANCED SEARCH # - # Accession Date - config.add_facet_field Settings.FIELDS.B1G_DATE_ACCESSIONED, :label => 'Date Accessioned', :limit => 8, collapse: false, sort: "index" # Code config.add_facet_field Settings.FIELDS.B1G_CODE, label: 'Code', show: false From ee776cc6cb1430ca0ac0ba0e8845d5ee5e6f1f8f Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Thu, 7 Dec 2023 14:47:53 -0600 Subject: [PATCH 07/20] Kithe: add BulkLoadingService --- app/controllers/catalog_controller.rb | 1 + app/models/solr_document.rb | 2 ++ 2 files changed, 3 insertions(+) diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb index 899a8c5a4..e49c79c49 100644 --- a/app/controllers/catalog_controller.rb +++ b/app/controllers/catalog_controller.rb @@ -5,6 +5,7 @@ class CatalogController < ApplicationController include BlacklightAdvancedSearch::Controller include BlacklightRangeLimit::ControllerOverride include Blacklight::Catalog + self.search_service_class = Kithe::BlacklightTools::BulkLoadingSearchService configure_blacklight do |config| # default advanced config values diff --git a/app/models/solr_document.rb b/app/models/solr_document.rb index ab6142dac..c102a2cca 100644 --- a/app/models/solr_document.rb +++ b/app/models/solr_document.rb @@ -6,6 +6,8 @@ class SolrDocument include WmsRewriteConcern include B1gLicensedDataConcern + attr_accessor :model + # self.unique_key = 'id' self.unique_key = Settings.FIELDS.B1G_GEOMG_ID From bf2ae85ab55f714d5851358242eeb2a6646446f4 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Thu, 7 Dec 2023 16:02:59 -0600 Subject: [PATCH 08/20] Sidekiq: turn down to two workers --- config/sidekiq.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/sidekiq.yml b/config/sidekiq.yml index ffa0268f4..580b18ed6 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -1,4 +1,4 @@ -:concurrency: 3 +:concurrency: 2 :max_retries: 1 :queues: From 1b858638ab02ec5c76a422e98fad95aa98db1f4c Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Thu, 7 Dec 2023 16:27:28 -0600 Subject: [PATCH 09/20] Update catalog_controller.rb Add Kithe require for bulk loading service --- app/controllers/catalog_controller.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/controllers/catalog_controller.rb b/app/controllers/catalog_controller.rb index e49c79c49..a94ff1c9b 100644 --- a/app/controllers/catalog_controller.rb +++ b/app/controllers/catalog_controller.rb @@ -1,5 +1,6 @@ # -*- encoding : utf-8 -*- require 'blacklight/catalog' +require 'kithe/blacklight_tools/bulk_loading_search_service' class CatalogController < ApplicationController include BlacklightAdvancedSearch::Controller From 8ae4580c1aa39dfbfb72875e59f888442460b64f Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Thu, 7 Dec 2023 16:46:38 -0600 Subject: [PATCH 10/20] Gems: bump gbl-admin --- Gemfile.lock | 51 +++++++++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 240954114..bb2927a53 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -41,7 +41,7 @@ GIT GIT remote: https://github.com/geobtaa/geoblacklight_admin.git - revision: d139c5e92480a25c28212dabd5233c7165522dfa + revision: 1e830d055823dfbb432777ea0e6dd468f18253bb branch: feature/bulk-actions-sti specs: geoblacklight_admin (0.3.2) @@ -119,7 +119,7 @@ GEM rails-html-sanitizer (~> 1.1, >= 1.2.0) active_record_query_trace (1.8.2) activerecord (>= 6.0.0) - active_storage_validations (1.0.4) + active_storage_validations (1.1.3) activejob (>= 5.2.0) activemodel (>= 5.2.0) activestorage (>= 5.2.0) @@ -158,9 +158,9 @@ GEM ansi (1.5.0) appsignal (3.4.12) rack - attr_json (2.2.0) + attr_json (2.3.0) activerecord (>= 6.0.0, < 7.2) - autoprefixer-rails (10.4.15.0) + autoprefixer-rails (10.4.16.0) execjs (~> 2) awesome_print (1.9.2) axe-core-api (4.7.0) @@ -170,9 +170,9 @@ GEM descendants_tracker (~> 0.0.4) ice_nine (~> 0.11.0) thread_safe (~> 0.3, >= 0.3.1) - base64 (0.1.1) + base64 (0.2.0) bcp47_spec (0.2.1) - bcrypt (3.1.19) + bcrypt (3.1.20) bindex (0.8.1) blacklight (7.33.1) deprecation @@ -252,7 +252,7 @@ GEM activerecord (>= 5.a) database_cleaner-core (~> 2.0.0) database_cleaner-core (2.0.1) - date (3.3.3) + date (3.3.4) deep_merge (1.2.2) deprecation (1.1.0) activesupport @@ -267,12 +267,11 @@ GEM devise-bootstrap-views (1.1.0) devise-guests (0.8.1) devise - devise_invitable (2.0.8) + devise_invitable (2.0.9) actionmailer (>= 5.0) devise (>= 4.6) diff-lcs (1.5.0) - domain_name (0.5.20190701) - unf (>= 0.0.5, < 1.0.0) + domain_name (0.6.20231109) dot-properties (0.1.4) bundler (>= 2.2.33) dotenv (2.8.1) @@ -326,7 +325,7 @@ GEM factory_bot_rails (6.2.0) factory_bot (~> 6.2.0) railties (>= 5.0.0) - faraday (2.7.11) + faraday (2.7.12) base64 faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) @@ -421,14 +420,14 @@ GEM activerecord kaminari-core (= 1.2.2) kaminari-core (1.2.2) - kithe (2.11.0) - attr_json (< 3.0.0) + kithe (2.13.0) + attr_json (~> 2.0) fastimage (~> 2.0) fx (>= 0.6.0, < 1) marcel mini_mime pdf-reader (~> 2.0) - rails (>= 5.2.1, < 7.1) + rails (>= 6.0, < 7.2) rsolr (~> 2.2) ruby-progressbar (~> 1.0) shrine (~> 3.3) @@ -452,7 +451,7 @@ GEM llhttp-ffi (0.3.1) ffi-compiler (~> 1.0) rake (~> 13.0) - loofah (2.21.4) + loofah (2.22.0) crass (~> 1.0.2) nokogiri (>= 1.12.0) m (1.5.1) @@ -475,7 +474,7 @@ GEM method_source (1.0.0) mime-types (3.5.1) mime-types-data (~> 3.2015) - mime-types-data (3.2023.1003) + mime-types-data (3.2023.1205) mimemagic (0.4.3) nokogiri (~> 1) rake @@ -495,24 +494,24 @@ GEM multi_xml (0.6.0) net-http-persistent (4.0.2) connection_pool (~> 2.2) - net-imap (0.4.2) + net-imap (0.4.7) date net-protocol net-pop (0.1.2) net-protocol - net-protocol (0.2.1) + net-protocol (0.2.2) timeout net-smtp (0.4.0) net-protocol - nio4r (2.5.9) - nokogiri (1.15.4) + nio4r (2.7.0) + nokogiri (1.15.5) mini_portile2 (~> 2.8.2) racc (~> 1.4) noticed (1.6.3) http (>= 4.0.0) rails (>= 5.2.0) orm_adapter (0.5.0) - ostruct (0.5.5) + ostruct (0.6.0) pagy (6.2.0) paper_trail (14.0.0) activerecord (>= 6.0) @@ -545,7 +544,7 @@ GEM nokogiri (~> 1.6) rails (>= 5.0, < 7.2) rdf - racc (1.7.1) + racc (1.7.3) rack (2.2.8) rack-cors (2.0.1) rack (>= 2.0.0) @@ -583,7 +582,7 @@ GEM method_source rake (>= 12.2) thor (~> 1.0) - rake (13.0.6) + rake (13.1.0) rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) @@ -602,7 +601,7 @@ GEM railties (>= 5.2) retriable (3.1.2) rexml (3.2.6) - rgeo (3.0.0) + rgeo (3.0.1) rgeo-geojson (2.1.1) rgeo (>= 1.0.0) rsolr (2.5.0) @@ -698,7 +697,7 @@ GEM thor (1.3.0) thread_safe (0.3.6) tilt (2.3.0) - timeout (0.4.0) + timeout (0.4.1) traject (3.8.1) concurrent-ruby (>= 0.8.0) dot-properties (>= 0.1.1) @@ -729,7 +728,7 @@ GEM execjs (>= 0.3.0, < 3) unf (0.1.4) unf_ext - unf_ext (0.0.8.2) + unf_ext (0.0.9.1) view_component (2.82.0) activesupport (>= 5.2.0, < 8.0) concurrent-ruby (~> 1.0) From b0a96a49fda312ac8e1d71a49b25ca5109f5ad62 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Thu, 7 Dec 2023 17:16:33 -0600 Subject: [PATCH 11/20] Update ids_controller.rb Add missing facets --- app/controllers/admin/ids_controller.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/controllers/admin/ids_controller.rb b/app/controllers/admin/ids_controller.rb index 6404dfb0c..b08d78f9b 100644 --- a/app/controllers/admin/ids_controller.rb +++ b/app/controllers/admin/ids_controller.rb @@ -125,7 +125,13 @@ class IdsController < ApplicationController } # Publication State - config.add_facet_field Settings.FIELDS.B1G_PUBLICATION_STATE, show: false + config.add_facet_field Settings.FIELDS.B1G_PUBLICATION_STATE, :label => 'Publication State', :limit => 8, collapse: false + + # Accession Date + config.add_facet_field Settings.FIELDS.B1G_DATE_ACCESSIONED, :label => 'Date Accessioned', :limit => 8, collapse: false + + # Import ID + config.add_facet_field Settings.FIELDS.B1G_IMPORT_ID, label: "Import ID", show: false # Resouce Class config.add_facet_field Settings.FIELDS.RESOURCE_CLASS, show: false From 1ad3ed699f81ec46ef784f84ff27cc9c32548bcd Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 8 Dec 2023 09:07:55 -0600 Subject: [PATCH 12/20] Update Gemfile.lock --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index bb2927a53..d232d91db 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -41,7 +41,7 @@ GIT GIT remote: https://github.com/geobtaa/geoblacklight_admin.git - revision: 1e830d055823dfbb432777ea0e6dd468f18253bb + revision: 750eaa3c25379adadbf478494c57bd9b88a50c73 branch: feature/bulk-actions-sti specs: geoblacklight_admin (0.3.2) From b27daf0c79d0290cc643b7ec2ce2e606bb45f5ea Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 8 Dec 2023 10:07:09 -0600 Subject: [PATCH 13/20] Update Gemfile.lock --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index d232d91db..e12352d4b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -41,7 +41,7 @@ GIT GIT remote: https://github.com/geobtaa/geoblacklight_admin.git - revision: 750eaa3c25379adadbf478494c57bd9b88a50c73 + revision: 772bf5226c90bbdc1ea0c787e95c7541d02ccc72 branch: feature/bulk-actions-sti specs: geoblacklight_admin (0.3.2) From 27c7a12caa26b70dfdc241c02afbcf89a024a995 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 8 Dec 2023 11:09:28 -0600 Subject: [PATCH 14/20] Update _footer_app.html.haml Remove Twitter --- app/views/shared/_footer_app.html.haml | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/views/shared/_footer_app.html.haml b/app/views/shared/_footer_app.html.haml index 735ff9207..be33cf2da 100644 --- a/app/views/shared/_footer_app.html.haml +++ b/app/views/shared/_footer_app.html.haml @@ -11,8 +11,6 @@ %a{href:'https://geo.btaa.org/feedback'} Contact Project Team %li %a{href:'http://www.btaa.org'} Big Ten Academic Alliance Homepage - %li - = link_to image_tag('twitter-logo.png', alt: 'Follow @geobtaa on Twitter') + ' Follow @geobtaa', 'https://twitter.com/geobtaa' %li %a#feedback{ href: "/feedback" } Feedback From b09ea94b82f11b049bf4f53cd44452518ec0f932 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 8 Dec 2023 11:17:28 -0600 Subject: [PATCH 15/20] Update search_results_test.rb Remove Twitter link --- test/system/search_results_test.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/test/system/search_results_test.rb b/test/system/search_results_test.rb index 9f4c1b436..cfc284107 100644 --- a/test/system/search_results_test.rb +++ b/test/system/search_results_test.rb @@ -23,7 +23,6 @@ def test_footer_nav assert page.has_link?("About Us") assert page.has_link?("Contact Project Team") assert page.has_link?("Big Ten Academic Alliance Homepage") - assert page.has_link?("Follow @geobtaa") end end From 7367f8fb335795028c3d2e442cac618e52d57bdd Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 8 Dec 2023 14:01:56 -0600 Subject: [PATCH 16/20] Tableau Export --- app/controllers/admin/api_controller.rb | 48 +++------------------- app/jobs/export_tableau_job.rb | 52 ++++++++++++++++++++++++ app/services/export_tableau_service.rb | 54 +++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 42 deletions(-) create mode 100644 app/jobs/export_tableau_job.rb create mode 100644 app/services/export_tableau_service.rb diff --git a/app/controllers/admin/api_controller.rb b/app/controllers/admin/api_controller.rb index a1021c316..e3cc75e91 100644 --- a/app/controllers/admin/api_controller.rb +++ b/app/controllers/admin/api_controller.rb @@ -345,48 +345,6 @@ class ApiController < ApplicationController config.autocomplete_path = 'suggest' end - # Admin CSV stream of Database data - def tableau_export - headers.delete("Content-Length") - headers["Cache-Control"] = "no-cache" - headers["Content-Type"] = "text/csv" - headers["Content-Disposition"] = "attachment; filename=\"geoportal_tableau_export.csv\"" - headers["X-Accel-Buffering"] = "no" - response.status = 200 - self.response_body = csv_enumerator - end - - def csv_enumerator - @csv_enumerator ||= Enumerator.new do |yielder| - yielder << CSV.generate_line([ - "Title", - "Provider", - "Resource Class", - "Resource Type", - "Index Year", - "Spatial Coverage", - "B1G Image", - "ID", - "Download", - "Language" - ]) - Document.find_each do |row| - yielder << CSV.generate_line([ - row.title, - row.schema_provider_s, - row.gbl_resourceClass_sm&.join("|"), - row.gbl_resourceType_sm&.join("|"), - row.gbl_indexYear_im&.join("|"), - row.dct_spatial_sm&.join("|"), - row.b1g_image_ss, - row.geomg_id_s, - row.dct_references_s.find { |ref| ref.category == "download" }&.value, - row.dct_language_sm&.join("|") - ]) - end - end - end - # Administrative view of document # - Sidecar Image # - URIs @@ -425,5 +383,11 @@ def advanced_search_facets end end end + + # Tableau Export + def tableau_export + ExportTableauJob.perform_later(current_user) + head :no_content + end end end diff --git a/app/jobs/export_tableau_job.rb b/app/jobs/export_tableau_job.rb new file mode 100644 index 000000000..168df1f42 --- /dev/null +++ b/app/jobs/export_tableau_job.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +require "csv" + +# ExportTableauJob +class ExportTableauJob < ApplicationJob + queue_as :priority + + def perform(current_user) + logger.debug("\n\n Background Job: ♞") + logger.debug("User: #{current_user.inspect}") + logger.debug("Export Service: Tableau Export") + logger.debug("\n\n") + + # Test broadcast + ActionCable.server.broadcast("export_channel", {data: "Hello from Export Tableau Job!"}) + + # Send progress + file_content = ExportTableauService.call + + # Write into tempfile + @tempfile = Tempfile.new(["export-#{Time.zone.today}", ".csv"]).tap do |file| + CSV.open(file, "wb") do |csv| + file_content.each do |row| + csv << row + end + end + end + + # Create notification + # Message: "Download Type|Row Count|Button Label" + notification = ExportNotification.with(message: "CSV (Tableau)|#{ActionController::Base.helpers.number_with_delimiter(file_content.size - 1)} rows|CSV") + + # Deliver notification + notification.deliver(current_user) + + # Attach CSV file (can only attach after persisted) + notification.record.file.attach(io: @tempfile, filename: "geomg-export-#{Time.zone.today}.csv", + content_type: "text/csv") + + # Update UI + ActionCable.server.broadcast("export_channel", { + data: "Notification ready!", + actions: [ + { + method: "RefreshNotifications", + payload: current_user.notifications.unread.count + } + ] + }) + end +end diff --git a/app/services/export_tableau_service.rb b/app/services/export_tableau_service.rb new file mode 100644 index 000000000..6cce043e9 --- /dev/null +++ b/app/services/export_tableau_service.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# ExportTableauService +class ExportTableauService + def self.call + ActionCable.server.broadcast("export_channel", {progress: 0}) + + total = Document.count + count = 0 + slice_count = 1000 + csv_file = [] + + Rails.logger.debug { "\n\nExportTableauService: #{total}\n\n" } + + csv_file << [ + "Title", + "Provider", + "Resource Class", + "Resource Type", + "Index Year", + "Spatial Coverage", + "B1G Image", + "ID", + "Download", + "Language" + ] + + Document.in_batches do |slice| + # Broadcast progress percentage + count += slice_count + progress = ((count.to_f / total) * 100).round + progress = 100 if progress > 100 + + slice.each do |row| + csv_file << [ + row.title, + row.schema_provider_s, + row.gbl_resourceClass_sm&.join("|"), + row.gbl_resourceType_sm&.join("|"), + row.gbl_indexYear_im&.join("|"), + row.dct_spatial_sm&.join("|"), + row.b1g_image_ss, + row.geomg_id_s, + row.dct_references_s.find { |ref| ref.category == "download" }&.value, + row.dct_language_sm&.join("|") + ] + end + + ActionCable.server.broadcast("export_channel", {progress: progress}) + end + + csv_file + end +end From 5174507f4974b81a5653ab75c4436e3decef4ae2 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 8 Dec 2023 14:04:08 -0600 Subject: [PATCH 17/20] Update Gemfile.lock --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index e12352d4b..7242697be 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -41,7 +41,7 @@ GIT GIT remote: https://github.com/geobtaa/geoblacklight_admin.git - revision: 772bf5226c90bbdc1ea0c787e95c7541d02ccc72 + revision: 4bf2ea3deae63e2bcbb81e133474e2b654bff9da branch: feature/bulk-actions-sti specs: geoblacklight_admin (0.3.2) From a862491a38cd2f7e0323b4c84df2b70a936ba5eb Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 8 Dec 2023 14:30:19 -0600 Subject: [PATCH 18/20] Gems: bump to GBL Admin latest --- Gemfile | 2 +- Gemfile.lock | 68 ++++++++++++++++++++++++---------------------------- 2 files changed, 32 insertions(+), 38 deletions(-) diff --git a/Gemfile b/Gemfile index 7b07890b3..be29819e0 100644 --- a/Gemfile +++ b/Gemfile @@ -80,7 +80,7 @@ gem 'bootstrap', '~> 4.0' gem 'popper_js' gem 'twitter-typeahead-rails', '0.11.1.pre.corejavascript' gem 'geoblacklight', '~> 4.1' -gem 'geoblacklight_admin', git: "https://github.com/geobtaa/geoblacklight_admin.git", branch: "feature/bulk-actions-sti" +gem 'geoblacklight_admin', "~> 0.4.0" gem "rubyzip", ">= 1.3.0" gem "awesome_print" diff --git a/Gemfile.lock b/Gemfile.lock index 7242697be..898b55316 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -39,42 +39,6 @@ GIT rails (>= 5.2, < 7.1) statesman (~> 10.0.0) -GIT - remote: https://github.com/geobtaa/geoblacklight_admin.git - revision: 4bf2ea3deae63e2bcbb81e133474e2b654bff9da - branch: feature/bulk-actions-sti - specs: - geoblacklight_admin (0.3.2) - active_storage_validations (~> 1.0) - amazing_print - blacklight (~> 7.33) - blacklight_advanced_search - blacklight_range_limit - bootstrap (~> 4.0) - chosen-rails (~> 1.10) - cocoon (~> 1.2) - config (~> 4.0) - devise (~> 4.7) - devise-bootstrap-views (~> 1.0) - devise_invitable (~> 2.0) - dotenv-rails (~> 2.8) - geoblacklight (~> 4.0) - haml (~> 5.2) - httparty (~> 0.21) - inline_svg (~> 1.9) - jquery-rails (~> 4.4) - kithe (~> 2.0) - noticed (~> 1.6) - pagy (~> 6.0) - paper_trail (~> 14.0) - pg (~> 1.4) - qa (~> 5.0) - rails (>= 6.1, < 7.1) - ruby-progressbar - simple_form (~> 5.0) - sprockets (< 4) - statesman (~> 10.0) - GEM remote: https://rubygems.org/ specs: @@ -363,6 +327,36 @@ GEM mime-types rails (>= 6.1, < 7.1) rgeo-geojson + geoblacklight_admin (0.4.0) + active_storage_validations (~> 1.0) + amazing_print + blacklight (~> 7.33) + blacklight_advanced_search + blacklight_range_limit + bootstrap (~> 4.0) + chosen-rails (~> 1.10) + cocoon (~> 1.2) + config (~> 4.0) + devise (~> 4.7) + devise-bootstrap-views (~> 1.0) + devise_invitable (~> 2.0) + dotenv-rails (~> 2.8) + geoblacklight (~> 4.0) + haml (~> 5.2) + httparty (~> 0.21) + inline_svg (~> 1.9) + jquery-rails (~> 4.4) + kithe (~> 2.0) + noticed (~> 1.6) + pagy (~> 6.0) + paper_trail (~> 14.0) + pg (~> 1.4) + qa (~> 5.0) + rails (>= 6.1, < 7.1) + ruby-progressbar + simple_form (~> 5.0) + sprockets (< 4) + statesman (~> 10.0) geocoder (1.8.2) git (1.11.0) rchardet (~> 1.8) @@ -796,7 +790,7 @@ DEPENDENCIES foreman geoblacklight (~> 4.1) geoblacklight-icons! - geoblacklight_admin! + geoblacklight_admin (~> 0.4.0) geoblacklight_sidecar_images! haml handlebars_assets (~> 0.23.0) From 256f63ba40d10f7c6aef2fd275d5348e4bb48287 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 8 Dec 2023 14:37:46 -0600 Subject: [PATCH 19/20] Links: update for gin.btaa.org site Fixes #536 --- app/views/_user_util_links.html.erb | 4 ++-- app/views/catalog/_home_text.html.erb | 2 +- app/views/shared/_footer_app.html.haml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/views/_user_util_links.html.erb b/app/views/_user_util_links.html.erb index 6a4c9ea56..9c2cbd340 100644 --- a/app/views/_user_util_links.html.erb +++ b/app/views/_user_util_links.html.erb @@ -2,7 +2,7 @@ <%= render_nav_actions do |config, action|%> <% end %> - + - + diff --git a/app/views/catalog/_home_text.html.erb b/app/views/catalog/_home_text.html.erb index a2d3b82fc..6d8597f02 100644 --- a/app/views/catalog/_home_text.html.erb +++ b/app/views/catalog/_home_text.html.erb @@ -78,7 +78,7 @@

The Big Ten Academic Alliance Geoportal provides discoverability and facilitates access to geospatial resources. The resources in the portal are selected and curated by librarians and geospatial specialists at twelve research institutions in the Big Ten Academic Alliance.

-

The resources include GIS datasets, web services, and digitized historical maps. Learn more about the research institutions involved and the sources of the geospatial records.

+

The resources include GIS datasets, web services, and digitized historical maps. Learn more about the research institutions involved and the sources of the geospatial records.

Follow Our Blog   Keep up to date with all the latest news.

diff --git a/app/views/shared/_footer_app.html.haml b/app/views/shared/_footer_app.html.haml index be33cf2da..5e9baa165 100644 --- a/app/views/shared/_footer_app.html.haml +++ b/app/views/shared/_footer_app.html.haml @@ -6,7 +6,7 @@ %li = link_to "News & Updates".html_safe, 'https://geobtaa.blogspot.com' %li - = link_to "About Us", 'https://sites.google.com/umn.edu/btaa-gdp/about' + = link_to "About Us", 'https://gin.btaa.org' %li %a{href:'https://geo.btaa.org/feedback'} Contact Project Team %li From e8e655be24ac6a99f1ef46a698e2e0918d47513f Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 8 Dec 2023 14:49:20 -0600 Subject: [PATCH 20/20] README: bump for 5.3.0 release --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a80ba5be7..1ddeba331 100644 --- a/README.md +++ b/README.md @@ -30,4 +30,4 @@ Visit the [project wiki](https://github.com/BTAA-Geospatial-Data-Project/geoport #### Release Version -B1G Geoportal Version v5.2.0 / GeoBlacklight v4.1.0 / GeoBlacklight Admin v0.3.2 +B1G Geoportal Version v5.3.0 / GeoBlacklight v4.1.0 / GeoBlacklight Admin v0.4.0