From c0a9ef92136749b941da6c280a550565f23e545d Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Mon, 16 Dec 2024 10:23:15 -0600 Subject: [PATCH] Feature - Decouple dct_references_s from document (#628) * Gems: bump to latest GBL Admin release --- Gemfile | 4 +- Gemfile.lock | 287 ++++++++-------- app/assets/stylesheets/geoportal/_todo.scss | 8 +- .../stylesheets/geoportal/geoportal.scss | 5 + app/channels/application_cable/connection.rb | 2 +- app/channels/export_channel.rb | 2 +- app/javascript/channels/export_channel.js | 10 +- app/javascript/channels/index.js | 7 +- app/javascript/entrypoints/application.js | 2 +- config/cable.yml | 13 +- config/environments/development.rb | 4 +- config/initializers/mime_types.rb | 1 + config/routes.rb | 322 ++++++++++-------- ...1009200524_create_admin_reference_types.rb | 13 + ...241010161420_create_document_references.rb | 14 + ...8823_rename_references_to_distributions.rb | 5 + db/schema.rb | 109 ++---- db/seeds.rb | 41 ++- docs/geodev_references_audit_11072024.txt | 145 ++++++++ docs/localhost_references_audit_11082024.txt | 229 +++++++++++++ docs/references_migration.txt | 21 ++ solr/conf/schema.xml | 3 + 22 files changed, 849 insertions(+), 398 deletions(-) create mode 100644 db/migrate/20241009200524_create_admin_reference_types.rb create mode 100644 db/migrate/20241010161420_create_document_references.rb create mode 100644 db/migrate/20241120238823_rename_references_to_distributions.rb create mode 100644 docs/geodev_references_audit_11072024.txt create mode 100644 docs/localhost_references_audit_11082024.txt create mode 100644 docs/references_migration.txt diff --git a/Gemfile b/Gemfile index 4c10ead59..971dd5b31 100644 --- a/Gemfile +++ b/Gemfile @@ -82,7 +82,7 @@ gem 'twitter-typeahead-rails', '0.11.1.pre.corejavascript' gem 'geoblacklight', '4.4' # GBL Admin -gem 'geoblacklight_admin', git: "https://github.com/geobtaa/geoblacklight_admin.git", branch: "develop" +gem 'geoblacklight_admin', "~> 0.6.0" gem 'git', ">= 1.13" gem "rubyzip", ">= 1.3.0" gem "awesome_print" @@ -104,7 +104,7 @@ gem 'geoblacklight_sidecar_images', git: "https://github.com/geoblacklight/geobl gem 'mini_magick', '~> 4.9.4' gem "image_processing", ">= 1.2" -gem 'statesman', '~> 11.0' +gem 'statesman', '>= 12.0' gem 'sidekiq', '~> 6.4' gem 'sidekiq-failures', '~> 1.0.0' gem 'down', '~> 5.0' diff --git a/Gemfile.lock b/Gemfile.lock index 896c9e6dd..8aa5987ed 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,10 +1,10 @@ GIT remote: https://github.com/cbeer/solr_wrapper.git - revision: 868986a97c76c7a50eeab835892c7471b07b0821 + revision: 4d9bac111bb9bc84e0deb810b9454e8a3bc59057 branch: master specs: solr_wrapper (4.1.0) - faraday + faraday (~> 2.5) faraday-follow_redirects minitar ostruct @@ -41,119 +41,81 @@ GIT rails (>= 5.2, < 8.0) statesman (>= 3.4) -GIT - remote: https://github.com/geobtaa/geoblacklight_admin.git - revision: 4fee2c640fa5778d2b854fb1744e09b6adcd586b - branch: develop - specs: - geoblacklight_admin (0.5.1) - active_storage_validations (~> 1.0) - amazing_print - blacklight (~> 7.33) - blacklight_advanced_search - blacklight_range_limit - bootstrap (~> 4.0) - chosen-rails (~> 1.10) - cocoon (~> 1.2) - config (~> 4.0) - devise (~> 4.7) - devise-bootstrap-views (~> 1.0) - dotenv-rails (~> 2.8) - geoblacklight (~> 4.4) - haml (~> 5.2) - httparty (~> 0.21) - inline_svg (~> 1.9) - jquery-rails (~> 4.4) - kithe (~> 2.0) - mutex_m (~> 0.2.0) - noticed (~> 1.6) - pagy (~> 6.0) - paper_trail (~> 15.0) - pg (~> 1.4) - qa (~> 5.0) - rails (~> 7.0, < 7.3) - ruby-progressbar - simple_form (~> 5.0) - sprockets (~> 3.0) - statesman (~> 11.0) - vite_rails (~> 3.0) - vite_ruby (>= 3.5) - zeitwerk (~> 2.6) - GEM remote: https://rubygems.org/ specs: Ascii85 (2.0.1) - actioncable (7.0.8.6) - actionpack (= 7.0.8.6) - activesupport (= 7.0.8.6) + actioncable (7.0.8.7) + actionpack (= 7.0.8.7) + activesupport (= 7.0.8.7) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (7.0.8.6) - actionpack (= 7.0.8.6) - activejob (= 7.0.8.6) - activerecord (= 7.0.8.6) - activestorage (= 7.0.8.6) - activesupport (= 7.0.8.6) + actionmailbox (7.0.8.7) + actionpack (= 7.0.8.7) + activejob (= 7.0.8.7) + activerecord (= 7.0.8.7) + activestorage (= 7.0.8.7) + activesupport (= 7.0.8.7) mail (>= 2.7.1) net-imap net-pop net-smtp - actionmailer (7.0.8.6) - actionpack (= 7.0.8.6) - actionview (= 7.0.8.6) - activejob (= 7.0.8.6) - activesupport (= 7.0.8.6) + actionmailer (7.0.8.7) + actionpack (= 7.0.8.7) + actionview (= 7.0.8.7) + activejob (= 7.0.8.7) + activesupport (= 7.0.8.7) mail (~> 2.5, >= 2.5.4) net-imap net-pop net-smtp rails-dom-testing (~> 2.0) - actionpack (7.0.8.6) - actionview (= 7.0.8.6) - activesupport (= 7.0.8.6) + actionpack (7.0.8.7) + actionview (= 7.0.8.7) + activesupport (= 7.0.8.7) rack (~> 2.0, >= 2.2.4) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (7.0.8.6) - actionpack (= 7.0.8.6) - activerecord (= 7.0.8.6) - activestorage (= 7.0.8.6) - activesupport (= 7.0.8.6) + actiontext (7.0.8.7) + actionpack (= 7.0.8.7) + activerecord (= 7.0.8.7) + activestorage (= 7.0.8.7) + activesupport (= 7.0.8.7) globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (7.0.8.6) - activesupport (= 7.0.8.6) + actionview (7.0.8.7) + activesupport (= 7.0.8.7) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.1, >= 1.2.0) active_record_query_trace (1.8.2) activerecord (>= 6.0.0) - active_storage_validations (1.3.0) + active_storage_validations (1.3.5) activejob (>= 6.1.4) activemodel (>= 6.1.4) activestorage (>= 6.1.4) activesupport (>= 6.1.4) - activejob (7.0.8.6) - activesupport (= 7.0.8.6) + marcel (>= 1.0.3) + activejob (7.0.8.7) + activesupport (= 7.0.8.7) globalid (>= 0.3.6) - activemodel (7.0.8.6) - activesupport (= 7.0.8.6) - activerecord (7.0.8.6) - activemodel (= 7.0.8.6) - activesupport (= 7.0.8.6) - activerecord-import (1.8.1) + activemodel (7.0.8.7) + activesupport (= 7.0.8.7) + activerecord (7.0.8.7) + activemodel (= 7.0.8.7) + activesupport (= 7.0.8.7) + activerecord-import (2.0.0) activerecord (>= 4.2) - activestorage (7.0.8.6) - actionpack (= 7.0.8.6) - activejob (= 7.0.8.6) - activerecord (= 7.0.8.6) - activesupport (= 7.0.8.6) + activestorage (7.0.8.7) + actionpack (= 7.0.8.7) + activejob (= 7.0.8.7) + activerecord (= 7.0.8.7) + activesupport (= 7.0.8.7) marcel (~> 1.0) mini_mime (>= 1.1.0) - activesupport (7.0.8.6) + activesupport (7.0.8.7) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -167,31 +129,31 @@ GEM net-http-persistent amazing_print (1.6.0) ansi (1.5.0) - appsignal (4.1.2) + appsignal (4.2.2) logger rack - attr_json (2.4.0) - activerecord (>= 6.0.0, < 7.3) + attr_json (2.5.0) + activerecord (>= 6.0.0, < 8.1) autoprefixer-rails (10.4.19.0) execjs (~> 2) awesome_print (1.9.2) aws-eventstream (1.3.0) - aws-partitions (1.1001.0) - aws-sdk-core (3.212.0) + aws-partitions (1.1023.0) + aws-sdk-core (3.214.0) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.992.0) aws-sigv4 (~> 1.9) jmespath (~> 1, >= 1.6.1) - aws-sdk-kms (1.95.0) + aws-sdk-kms (1.96.0) aws-sdk-core (~> 3, >= 3.210.0) aws-sigv4 (~> 1.5) - aws-sdk-s3 (1.170.0) + aws-sdk-s3 (1.176.1) aws-sdk-core (~> 3, >= 3.210.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.5) aws-sigv4 (1.10.1) aws-eventstream (~> 1, >= 1.0.2) - axe-core-api (4.10.1) + axe-core-api (4.10.2) dumb_delegator ostruct virtus @@ -276,14 +238,14 @@ GEM connection_pool (2.4.1) content_disposition (1.0.0) crass (1.0.6) - csv (3.3.0) + csv (3.3.1) database_cleaner (2.1.0) database_cleaner-active_record (>= 2, < 3) database_cleaner-active_record (2.2.0) activerecord (>= 5.a) database_cleaner-core (~> 2.0.0) database_cleaner-core (2.0.1) - date (3.4.0) + date (3.4.1) deep_merge (1.2.2) deprecation (1.1.0) activesupport @@ -357,14 +319,14 @@ GEM factory_bot_rails (6.4.4) factory_bot (~> 6.5) railties (>= 5.0.0) - faraday (2.12.0) - faraday-net_http (>= 2.0, < 3.4) + faraday (2.12.2) + faraday-net_http (>= 2.0, < 3.5) json logger faraday-follow_redirects (0.3.0) faraday (>= 1, < 3) - faraday-net_http (3.3.0) - net-http + faraday-net_http (3.4.0) + net-http (>= 0.5.0) faraday-net_http_persistent (2.3.0) faraday (~> 2.5) net-http-persistent (>= 4.0.4, < 5) @@ -377,9 +339,9 @@ GEM rake flamegraph (0.9.5) foreman (0.88.1) - fx (0.8.0) - activerecord (>= 6.0.0) - railties (>= 6.0.0) + fx (0.9.0) + activerecord (>= 7.0) + railties (>= 7.0) geo_combine (0.9.2) activesupport faraday-net_http_persistent (~> 2.0) @@ -403,10 +365,43 @@ GEM rgeo-geojson sprockets-rails (~> 3.0) vite_rails (~> 3.0) - geocoder (1.8.3) + geoblacklight_admin (0.6.1) + active_storage_validations (~> 1.0) + amazing_print + blacklight (~> 7.33) + blacklight_advanced_search + blacklight_range_limit + bootstrap (~> 4.0) + chosen-rails (~> 1.10) + cocoon (~> 1.2) + config (~> 4.0) + devise (~> 4.7) + devise-bootstrap-views (~> 1.0) + dotenv-rails (~> 2.8) + geoblacklight (~> 4.4) + haml (~> 5.2) + httparty (~> 0.21) + inline_svg (~> 1.9) + jquery-rails (~> 4.4) + kithe (~> 2.0) + mutex_m (~> 0.2.0) + noticed (~> 1.6) + pagy (~> 6.0) + paper_trail (~> 15.0) + pg (~> 1.4) + qa (~> 5.0) + rails (~> 7.0, < 7.3) + ruby-progressbar + simple_form (~> 5.0) + sprockets (~> 3.0) + statesman (~> 12.0) + vite_rails (~> 3.0) + vite_ruby (>= 3.5) + zeitwerk (~> 2.6) + geocoder (1.8.4) base64 (>= 0.1.0) csv (>= 3.0.0) - git (2.3.1) + git (2.3.3) activesupport (>= 5.0) addressable (~> 2.8) process_executer (~> 1.1) @@ -420,7 +415,7 @@ GEM execjs (~> 2.0) sprockets (>= 2.0.0) tilt (>= 1.2) - hashdiff (1.1.1) + hashdiff (1.1.2) hashery (2.1.2) hashie (5.0.0) http (5.2.0) @@ -429,7 +424,7 @@ GEM http-cookie (~> 1.0) http-form_data (~> 2.2) llhttp-ffi (~> 0.5.0) - http-cookie (1.0.7) + http-cookie (1.0.8) domain_name (~> 0.5) http-form_data (2.3.0) httparty (0.22.0) @@ -454,9 +449,10 @@ GEM rails-dom-testing (>= 1, < 3) railties (>= 4.2.0) thor (>= 0.14, < 2.0) - json (2.8.1) - json-schema (5.0.1) + json (2.9.0) + json-schema (5.1.1) addressable (~> 2.8) + bigdecimal (~> 3.1) kaminari (1.2.2) activesupport (>= 4.1.0) kaminari-actionview (= 1.2.2) @@ -469,14 +465,14 @@ GEM activerecord kaminari-core (= 1.2.2) kaminari-core (1.2.2) - kithe (2.15.1) + kithe (2.16.0) attr_json (~> 2.0) fastimage (~> 2.0) fx (>= 0.6.0, < 1) marcel mini_mime pdf-reader (~> 2.0) - rails (>= 6.0, < 8.0) + rails (>= 6.0, < 8.1) rsolr (~> 2.2) ruby-progressbar (~> 1.0) shrine (~> 3.3) @@ -501,7 +497,7 @@ GEM llhttp-ffi (0.5.0) ffi-compiler (~> 1.0) rake (~> 13.0) - logger (1.6.1) + logger (1.6.3) loofah (2.23.1) crass (~> 1.0.2) nokogiri (>= 1.12.0) @@ -526,15 +522,15 @@ GEM mime-types (3.6.0) logger mime-types-data (~> 3.2015) - mime-types-data (3.2024.1105) + mime-types-data (3.2024.1203) mimemagic (0.4.3) nokogiri (~> 1) rake mini_magick (4.9.5) mini_mime (1.1.5) - mini_portile2 (2.8.7) + mini_portile2 (2.8.8) minitar (1.0.2) - minitest (5.25.1) + minitest (5.25.4) minitest-ci (3.4.0) minitest (>= 5.0.6) minitest-reporters (1.7.1) @@ -542,7 +538,7 @@ GEM builder minitest (>= 5.0) ruby-progressbar - msgpack (1.7.3) + msgpack (1.7.5) multi_json (1.15.0) multi_xml (0.7.1) bigdecimal (~> 3.1) @@ -550,11 +546,11 @@ GEM net-ftp (0.3.8) net-protocol time - net-http (0.4.1) + net-http (0.6.0) uri - net-http-persistent (4.0.4) + net-http-persistent (4.0.5) connection_pool (~> 2.2) - net-imap (0.5.0) + net-imap (0.5.2) date net-protocol net-pop (0.1.2) @@ -564,14 +560,14 @@ GEM net-smtp (0.5.0) net-protocol nio4r (2.7.4) - nokogiri (1.16.7) + nokogiri (1.17.2) mini_portile2 (~> 2.8.2) racc (~> 1.4) noticed (1.6.3) http (>= 4.0.0) rails (>= 5.2.0) orm_adapter (0.5.0) - ostruct (0.6.0) + ostruct (0.6.1) pagy (6.5.0) paper_trail (15.2.0) activerecord (>= 6.1) @@ -596,14 +592,14 @@ GEM public_suffix (6.0.1) puma (5.6.9) nio4r (~> 2.0) - qa (5.13.0) + qa (5.14.0) activerecord-import deprecation faraday (< 3.0, != 2.0.0) geocoder ldpath nokogiri (~> 1.6) - rails (>= 5.0, < 8.0) + rails (>= 5.0, < 8.1) rdf racc (1.8.1) rack (2.2.10) @@ -615,30 +611,30 @@ GEM rack rack-test (2.1.0) rack (>= 1.3) - rails (7.0.8.6) - actioncable (= 7.0.8.6) - actionmailbox (= 7.0.8.6) - actionmailer (= 7.0.8.6) - actionpack (= 7.0.8.6) - actiontext (= 7.0.8.6) - actionview (= 7.0.8.6) - activejob (= 7.0.8.6) - activemodel (= 7.0.8.6) - activerecord (= 7.0.8.6) - activestorage (= 7.0.8.6) - activesupport (= 7.0.8.6) + rails (7.0.8.7) + actioncable (= 7.0.8.7) + actionmailbox (= 7.0.8.7) + actionmailer (= 7.0.8.7) + actionpack (= 7.0.8.7) + actiontext (= 7.0.8.7) + actionview (= 7.0.8.7) + activejob (= 7.0.8.7) + activemodel (= 7.0.8.7) + activerecord (= 7.0.8.7) + activestorage (= 7.0.8.7) + activesupport (= 7.0.8.7) bundler (>= 1.15.0) - railties (= 7.0.8.6) + railties (= 7.0.8.7) rails-dom-testing (2.2.0) activesupport (>= 5.0.0) minitest nokogiri (>= 1.6) - rails-html-sanitizer (1.6.0) + rails-html-sanitizer (1.6.2) loofah (~> 2.21) - nokogiri (~> 1.14) - railties (7.0.8.6) - actionpack (= 7.0.8.6) - activesupport (= 7.0.8.6) + nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) + railties (7.0.8.7) + actionpack (= 7.0.8.7) + activesupport (= 7.0.8.7) method_source rake (>= 12.2) thor (~> 1.0) @@ -655,14 +651,14 @@ GEM rdf-vocab (3.3.2) rdf (~> 3.3) redis (4.8.1) - regexp_parser (2.9.2) + regexp_parser (2.9.3) request_store (1.7.0) rack (>= 1.4) responders (3.1.1) actionpack (>= 5.2) railties (>= 5.2) retriable (3.1.2) - rexml (3.3.9) + rexml (3.4.0) rgeo (3.0.1) rgeo-geojson (2.2.0) multi_json (~> 1.15) @@ -718,7 +714,7 @@ GEM tilt scrub_rb (1.0.1) sd_notify (0.1.1) - selenium-webdriver (4.26.0) + selenium-webdriver (4.27.0) base64 (~> 0.2) logger (~> 1.4) rexml (~> 3.2, >= 3.2.5) @@ -762,7 +758,7 @@ GEM sqlite3 (1.7.3) mini_portile2 (~> 2.8.0) stackprof (0.2.26) - statesman (11.0.0) + statesman (12.1.0) stimulus-rails (1.3.4) railties (>= 6.0.0) temple (0.10.3) @@ -773,9 +769,9 @@ GEM thor (1.3.2) thread_safe (0.3.6) tilt (2.4.0) - time (0.4.0) + time (0.4.1) date - timeout (0.4.2) + timeout (0.4.3) traject (3.8.2) concurrent-ruby (>= 0.8.0) dot-properties (>= 0.1.1) @@ -804,7 +800,7 @@ GEM tzinfo (2.0.6) concurrent-ruby (~> 1.0) unf (0.2.0) - uri (0.13.1) + uri (1.0.2) view_component (2.83.0) activesupport (>= 5.2.0, < 8.0) concurrent-ruby (~> 1.0) @@ -816,9 +812,10 @@ GEM vite_rails (3.0.19) railties (>= 5.1, < 9) vite_ruby (~> 3.0, >= 3.2.2) - vite_ruby (3.9.0) + vite_ruby (3.9.1) dry-cli (>= 0.7, < 2) logger (~> 1.6) + mutex_m rack-proxy (~> 0.6, >= 0.6.1) zeitwerk (~> 2.2) warden (1.2.9) @@ -881,7 +878,7 @@ DEPENDENCIES foreman geoblacklight (= 4.4) geoblacklight-icons! - geoblacklight_admin! + geoblacklight_admin (~> 0.6.0) geoblacklight_sidecar_images! git (>= 1.13) haml @@ -925,7 +922,7 @@ DEPENDENCIES spring sqlite3 (~> 1.4) stackprof (~> 0.2.12) - statesman (~> 11.0) + statesman (>= 12.0) stimulus-rails terser turbolinks diff --git a/app/assets/stylesheets/geoportal/_todo.scss b/app/assets/stylesheets/geoportal/_todo.scss index e0dc93e2d..d57c7d50a 100644 --- a/app/assets/stylesheets/geoportal/_todo.scss +++ b/app/assets/stylesheets/geoportal/_todo.scss @@ -24,7 +24,13 @@ a { border-color: $visited-color; } } -a.btn { text-decoration: none; } +a.btn { + text-decoration: none; + &:visited { + color: $white; + border-color: $white; + } +} a.btn.btn-primary { color: $white; background-color: $btaa-blue; diff --git a/app/assets/stylesheets/geoportal/geoportal.scss b/app/assets/stylesheets/geoportal/geoportal.scss index 1ca830585..2ed7a2c0e 100644 --- a/app/assets/stylesheets/geoportal/geoportal.scss +++ b/app/assets/stylesheets/geoportal/geoportal.scss @@ -22,3 +22,8 @@ .pull-right { float: right; } + +// Fix input#q in search form +input#q { + background-color: $white !important; +} diff --git a/app/channels/application_cable/connection.rb b/app/channels/application_cable/connection.rb index 2c84e59bf..e59083764 100644 --- a/app/channels/application_cable/connection.rb +++ b/app/channels/application_cable/connection.rb @@ -20,4 +20,4 @@ def find_verified_user end end end -end +end \ No newline at end of file diff --git a/app/channels/export_channel.rb b/app/channels/export_channel.rb index 742084409..ee9bf4b5f 100644 --- a/app/channels/export_channel.rb +++ b/app/channels/export_channel.rb @@ -9,4 +9,4 @@ def subscribed def unsubscribed # Any cleanup needed when channel is unsubscribed end -end +end \ No newline at end of file diff --git a/app/javascript/channels/export_channel.js b/app/javascript/channels/export_channel.js index b58579644..ea90f3ac6 100644 --- a/app/javascript/channels/export_channel.js +++ b/app/javascript/channels/export_channel.js @@ -1,18 +1,18 @@ import consumer from "./consumer" -consumer.subscriptions.create({ channel: "ExportChannel" }, { +export default consumer.subscriptions.create({ channel: "ExportChannel" }, { connected() { // Called when the subscription is ready for use on the server - console.log('Export Channel Connected'); + console.log("GBL Admin - ExportChannel connected"); }, disconnected() { // Called when the subscription has been terminated by the server - console.log('Export Channel Disconnected'); + console.log("GBL Admin - ExportChannel disconnected"); }, received(data) { - console.log('Export Channel Received'); + console.log('GBL Admin - ExportChannel received!'); console.log(data); if (data['progress']) { @@ -27,4 +27,4 @@ consumer.subscriptions.create({ channel: "ExportChannel" }, { } } } -}); +}); \ No newline at end of file diff --git a/app/javascript/channels/index.js b/app/javascript/channels/index.js index 0cfcf7491..30454868f 100644 --- a/app/javascript/channels/index.js +++ b/app/javascript/channels/index.js @@ -1,5 +1,2 @@ -// Load all the channels within this directory and all subdirectories. -// Channel files must be named *_channel.js. - -const channels = require.context('.', true, /_channel\.js$/) -channels.keys().forEach(channels) +export { default as consumer } from './consumer'; +export { default as exportChannel } from './export_channel'; diff --git a/app/javascript/entrypoints/application.js b/app/javascript/entrypoints/application.js index 619e35528..9dec89a2a 100644 --- a/app/javascript/entrypoints/application.js +++ b/app/javascript/entrypoints/application.js @@ -22,7 +22,7 @@ console.log('Vite ⚡️ Rails') // ActiveStorage.start() // // // Import all channels. -// const channels = import.meta.globEager('./**/*_channel.js') +import '../channels'; // Example: Import a stylesheet in app/frontend/index.css // import '~/index.css' diff --git a/config/cable.yml b/config/cable.yml index 232fc5be8..b03f164f4 100644 --- a/config/cable.yml +++ b/config/cable.yml @@ -1,10 +1,9 @@ +# config/cable.yml +production: + adapter: postgresql + development: - adapter: async + adapter: postgresql test: - adapter: async - -production: - adapter: redis - url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %> - channel_prefix: geoblacklight_production + adapter: postgresql \ No newline at end of file diff --git a/config/environments/development.rb b/config/environments/development.rb index 3d3f7f715..a80ac11cc 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -64,12 +64,12 @@ # Raises error for missing translations. # config.i18n.raise_on_missing_translations = true - # config.active_job.queue_adapter = :inline config.active_job.queue_adapter = :sidekiq + Redis.exists_returns_integer = true # Annotate rendered view with file names. # config.action_view.annotate_rendered_view_with_filenames = true # Uncomment if you wish to allow Action Cable access from any origin. - # config.action_cable.disable_request_forgery_protection = true + config.action_cable.disable_request_forgery_protection = true end diff --git a/config/initializers/mime_types.rb b/config/initializers/mime_types.rb index 5b678f929..3426ef927 100644 --- a/config/initializers/mime_types.rb +++ b/config/initializers/mime_types.rb @@ -13,3 +13,4 @@ Mime::Type.register "text/csv", :csv_document_downloads Mime::Type.register "text/csv", :csv_document_access_links +Mime::Type.register "text/csv", :csv_document_distributions diff --git a/config/routes.rb b/config/routes.rb index e3f928b50..6a5eae6f9 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,4 +1,6 @@ Rails.application.routes.draw do + mount ActionCable.server => "/cable" + get 'about', :to => redirect('https://gin.btaa.org/') get 'help', :to => redirect('https://gin.btaa.org/guides/') get 'robots.:format' => 'robots#robots' @@ -78,147 +80,118 @@ end end - #################### - # GBL‡ADMIN +#################### +# GBL‡ADMIN - # Bulk Actions - resources :bulk_actions do - patch :run, on: :member - patch :revert, on: :member - end +# Bulk Actions +resources :bulk_actions do + patch :run, on: :member + patch :revert, on: :member +end - # Users - devise_for :users, controllers: {invitations: "devise/invitations"}, skip: [:registrations] - as :user do - get "/sign_in" => "devise/sessions#new" # custom path to login/sign_in - get "/sign_up" => "devise/registrations#new", :as => "new_user_registration" # custom path to sign_up/registration - get "users/edit" => "devise/registrations#edit", :as => "edit_user_registration" - put "users" => "devise/registrations#update", :as => "user_registration" - end +# Users +devise_for :users, skip: [:registrations] +as :user do + get "/sign_in" => "devise/sessions#new" # custom path to login/sign_in + get "/sign_up" => "devise/registrations#new", :as => "new_user_registration" # custom path to sign_up/registration + get "users/edit" => "devise/registrations#edit", :as => "edit_user_registration" + put "users" => "devise/registrations#update", :as => "user_registration" +end - namespace :admin do +namespace :admin do - authenticate :user, ->(user) { user.admin? } do - mount Sidekiq::Web => "/sidekiq" - end - - devise_for :users, controllers: {invitations: "devise/invitations"}, skip: [:registrations] - # Root - root to: "documents#index" - - # Assets - # Note "assets" is Rails reserved word for routing, oops. So we use - # asset_files. - resources :assets, path: "asset_files" do - collection do - get "display_attach_form" - post "attach_files" + authenticate :user, ->(user) { user.admin? } do + mount Sidekiq::Web => "/sidekiq" + end - get "destroy_all" - post "destroy_all" - end + devise_for :users - post :sort, on: :collection - end + # Root + root to: "documents#index" - # Bulk Actions - resources :bulk_actions do - patch :run, on: :member - patch :revert, on: :member - end + # Assets + # Note "assets" is Rails reserved word for routing, oops. So we use + # asset_files. + resources :assets, path: "asset_files" do + collection do + get "display_attach_form" + post "attach_files" - # Imports - resources :imports do - resources :mappings - resources :import_documents, only: [:show] - patch :run, on: :member + get "destroy_all" + post "destroy_all" end - # Elements - resources :elements do - post :sort, on: :collection - end + post :sort, on: :collection + end - # Form Elements - resources :form_elements do - post :sort, on: :collection - end - resources :form_header, path: :form_elements, controller: :form_elements - resources :form_group, path: :form_elements, controller: :form_elements - resources :form_control, path: :form_elements, controller: :form_elements - resources :form_feature, path: :form_elements, controller: :form_elements - - # Notifications - resources :notifications do - put "batch", on: :collection - end + # Bulk Actions + resources :bulk_actions do + patch :run, on: :member + patch :revert, on: :member + end - # Users - get "users/index" - - # Bookmarks - resources :bookmarks - delete "/bookmarks", to: "bookmarks#destroy", as: :bookmarks_destroy_by_fkeys - - # Search controller - get "/search" => "search#index" - - # AdvancedSearch controller - get '/advanced_search' => 'advanced_search#index', constraints: lambda { |req| req.format == :json } - get '/advanced_search/facets' => 'advanced_search#facets', constraints: lambda { |req| req.format == :json } - get '/advanced_search/facet/:id' => 'advanced_search#facet', constraints: lambda { |req| req.format == :json }, as: 'advanced_search_facet' - - # Ids controller - get '/api/ids' => 'ids#index', constraints: lambda { |req| req.format == :json } - get '/api' => 'api#index', constraints: lambda { |req| req.format == :json } - get '/api/fetch' => 'api#fetch', constraints: lambda { |req| req.format == :json } - get '/api/facet/:id' => 'api#facet', constraints: lambda { |req| req.format == :json } - get '/api/tableau_export' => 'api#tableau_export' - - # Documents - resources :documents do - get "admin" - get "versions" - - # DocumentAccesses - resources :document_accesses, path: "access" do - collection do - get "import" - post "import" - - get "destroy_all" - post "destroy_all" - end - end + # Imports + resources :imports do + resources :mappings + resources :import_documents, only: [:show] + patch :run, on: :member + end - # DocumentDownloads - resources :document_downloads, path: "downloads" do - collection do - get "import" - post "import" + # Elements + resources :elements do + post :sort, on: :collection + end - get "destroy_all" - post "destroy_all" - end - end + # Form Elements + resources :form_elements do + post :sort, on: :collection + end + resources :form_header, path: :form_elements, controller: :form_elements + resources :form_group, path: :form_elements, controller: :form_elements + resources :form_control, path: :form_elements, controller: :form_elements + resources :form_feature, path: :form_elements, controller: :form_elements + + # Reference Types + resources :reference_types do + post :sort, on: :collection + end - # Document Assets - resources :document_assets, path: "assets" do - collection do - get "display_attach_form" - post "attach_files" + # Notifications + resources :notifications do + put "batch", on: :collection + end - get "destroy_all" - post "destroy_all" - end - end + # Users + get "users/index" + + # Bookmarks + resources :bookmarks + delete "/bookmarks", to: "bookmarks#destroy", as: :bookmarks_destroy_by_fkeys + + # Search controller + get "/search" => "search#index" + + # AdvancedSearch controller + get '/advanced_search' => 'advanced_search#index', constraints: lambda { |req| req.format == :json } + get '/advanced_search/facets' => 'advanced_search#facets', constraints: lambda { |req| req.format == :json } + get '/advanced_search/facet/:id' => 'advanced_search#facet', constraints: lambda { |req| req.format == :json }, as: 'advanced_search_facet' + + # Ids controller + get '/api/ids' => 'ids#index', constraints: lambda { |req| req.format == :json } + get '/api' => 'api#index', constraints: lambda { |req| req.format == :json } + get '/api/fetch' => 'api#fetch', constraints: lambda { |req| req.format == :json } + get '/api/facet/:id' => 'api#facet', constraints: lambda { |req| req.format == :json } + + # Documents + resources :documents do + get "admin" + get "versions" - collection do - get "fetch" - end + collection do + get "fetch" end - # Document Accesses + # DocumentAccesses resources :document_accesses, path: "access" do collection do get "import" @@ -229,7 +202,18 @@ end end - # Document Downloads + # Document Assets + resources :document_assets, path: "assets" do + collection do + get "display_attach_form" + post "attach_files" + + get "destroy_all" + post "destroy_all" + end + end + + # DocumentDownloads resources :document_downloads, path: "downloads" do collection do get "import" @@ -240,40 +224,88 @@ end end - # Document Assets - resources :document_assets, path: "assets" do + # Document Distributions + resources :document_distributions, path: "distributions" do collection do get "display_attach_form" post "attach_files" + get "import" + post "import" + get "destroy_all" post "destroy_all" end end + end - # Assets - get "/asset_files/ingest", to: "assets#display_attach_form", as: "assets_ingest" - post "/asset_files/ingest", to: "assets#attach_files" - - # DocumentAssets - get "/documents/:id/ingest", to: "document_assets#display_attach_form", as: "asset_ingest" - post "/documents/:id/ingest", to: "document_assets#attach_files" - mount Kithe::AssetUploader.upload_endpoint(:cache) => "/direct_upload", :as => :direct_app_upload - - resources :collections, except: [:show] - - # Note "assets" is Rails reserved word for routing, oops. So we use - # asset_files. - resources :assets, path: "asset_files", except: [:new, :create] do - member do - put "convert_to_child_work" - end + # Document Accesses + resources :document_accesses, path: "access" do + collection do + get "import" + post "import" + + get "destroy_all" + post "destroy_all" end + end - # @TODO - # mount Qa::Engine => "/authorities" - mount ActionCable.server => "/cable" + # Document Downloads + resources :document_downloads, path: "downloads" do + collection do + get "import" + post "import" + + get "destroy_all" + post "destroy_all" + end + end + + # Document Distributions + resources :document_distributions, path: "distributions" do + collection do + get "import" + post "import" + + get "destroy_all" + post "destroy_all" + end + end + + # Document Assets + resources :document_assets, path: "assets" do + collection do + get "display_attach_form" + post "attach_files" + + get "destroy_all" + post "destroy_all" + end + end + + # Assets + get "/asset_files/ingest", to: "assets#display_attach_form", as: "assets_ingest" + post "/asset_files/ingest", to: "assets#attach_files" + + # DocumentAssets + get "/documents/:id/ingest", to: "document_assets#display_attach_form", as: "asset_ingest" + post "/documents/:id/ingest", to: "document_assets#attach_files" + + # Asset Direct Upload + mount Kithe::AssetUploader.upload_endpoint(:cache) => "/direct_upload", :as => :direct_app_upload + + resources :collections, except: [:show] + + # Note "assets" is Rails reserved word for routing, oops. So we use + # asset_files. + resources :assets, path: "asset_files", except: [:new, :create] do + member do + put "convert_to_child_work" + end + end + # @TODO + # mount Qa::Engine => "/authorities" # @TODO authenticate :user, ->(user) { user } do mount Blazer::Engine, at: "blazer" diff --git a/db/migrate/20241009200524_create_admin_reference_types.rb b/db/migrate/20241009200524_create_admin_reference_types.rb new file mode 100644 index 000000000..4a68e1476 --- /dev/null +++ b/db/migrate/20241009200524_create_admin_reference_types.rb @@ -0,0 +1,13 @@ +class CreateAdminReferenceTypes < ActiveRecord::Migration[7.0] + def change + create_table :reference_types do |t| + t.string :name # short name, ex: "cog" + t.string :reference_type # human-readable name, ex: "Cloud Optimized GeoTIFF (COG)" + t.string :reference_uri # key name, ex: "https://github.com/cogeotiff/cog-spec" + t.boolean :label, default: false # optional download label + t.text :note # optional note + t.integer :position # position for sorting + t.timestamps + end + end +end \ No newline at end of file diff --git a/db/migrate/20241010161420_create_document_references.rb b/db/migrate/20241010161420_create_document_references.rb new file mode 100644 index 000000000..458a18399 --- /dev/null +++ b/db/migrate/20241010161420_create_document_references.rb @@ -0,0 +1,14 @@ +class CreateDocumentReferences < ActiveRecord::Migration[7.0] + def change + create_table :document_references do |t| + t.string :friendlier_id, null: false + t.references :reference_type, null: false, foreign_key: true + t.string :url + t.string :label + t.integer :position + t.timestamps + + t.index [:friendlier_id, :reference_type_id, :url], unique: true, name: 'document_references_unique_index' + end + end +end \ No newline at end of file diff --git a/db/migrate/20241120238823_rename_references_to_distributions.rb b/db/migrate/20241120238823_rename_references_to_distributions.rb new file mode 100644 index 000000000..7bf4153ac --- /dev/null +++ b/db/migrate/20241120238823_rename_references_to_distributions.rb @@ -0,0 +1,5 @@ +class RenameReferencesToDistributions < ActiveRecord::Migration[7.0] + def change + rename_table :document_references, :document_distributions + end +end \ No newline at end of file diff --git a/db/schema.rb b/db/schema.rb index 651a13d95..c59d94473 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2024_06_19_183528) do +ActiveRecord::Schema[7.0].define(version: 2024_11_20_238823) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -214,13 +214,11 @@ t.string "name" t.string "request", null: false t.string "scope", null: false - t.string "field_name" - t.string "field_value" + t.string "field_name", null: false + t.string "field_value", null: false t.text "notes" t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.string "action_type", default: "ChangePublicationState", null: false - t.string "action" end create_table "document_accesses", force: :cascade do |t| @@ -231,6 +229,18 @@ t.datetime "updated_at", null: false end + create_table "document_distributions", force: :cascade do |t| + t.string "friendlier_id", null: false + t.bigint "reference_type_id", null: false + t.string "url" + t.string "label" + t.integer "position" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["friendlier_id", "reference_type_id", "url"], name: "document_references_unique_index", unique: true + t.index ["reference_type_id"], name: "index_document_distributions_on_reference_type_id" + end + create_table "document_downloads", force: :cascade do |t| t.string "friendlier_id" t.string "label" @@ -301,83 +311,6 @@ t.datetime "updated_at", null: false end - create_table "good_job_batches", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.text "description" - t.jsonb "serialized_properties" - t.text "on_finish" - t.text "on_success" - t.text "on_discard" - t.text "callback_queue_name" - t.integer "callback_priority" - t.datetime "enqueued_at", precision: nil - t.datetime "discarded_at", precision: nil - t.datetime "finished_at", precision: nil - end - - create_table "good_job_executions", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.uuid "active_job_id", null: false - t.text "job_class" - t.text "queue_name" - t.jsonb "serialized_params" - t.datetime "scheduled_at", precision: nil - t.datetime "finished_at", precision: nil - t.text "error" - t.integer "error_event", limit: 2 - t.index ["active_job_id", "created_at"], name: "index_good_job_executions_on_active_job_id_and_created_at" - end - - create_table "good_job_processes", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.jsonb "state" - end - - create_table "good_job_settings", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.text "key" - t.jsonb "value" - t.index ["key"], name: "index_good_job_settings_on_key", unique: true - end - - create_table "good_jobs", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| - t.text "queue_name" - t.integer "priority" - t.jsonb "serialized_params" - t.datetime "scheduled_at", precision: nil - t.datetime "performed_at", precision: nil - t.datetime "finished_at", precision: nil - t.text "error" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.uuid "active_job_id" - t.text "concurrency_key" - t.text "cron_key" - t.uuid "retried_good_job_id" - t.datetime "cron_at", precision: nil - t.uuid "batch_id" - t.uuid "batch_callback_id" - t.boolean "is_discrete" - t.integer "executions_count" - t.text "job_class" - t.integer "error_event", limit: 2 - t.index ["active_job_id", "created_at"], name: "index_good_jobs_on_active_job_id_and_created_at" - t.index ["active_job_id"], name: "index_good_jobs_on_active_job_id" - t.index ["batch_callback_id"], name: "index_good_jobs_on_batch_callback_id", where: "(batch_callback_id IS NOT NULL)" - t.index ["batch_id"], name: "index_good_jobs_on_batch_id", where: "(batch_id IS NOT NULL)" - t.index ["concurrency_key"], name: "index_good_jobs_on_concurrency_key_when_unfinished", where: "(finished_at IS NULL)" - t.index ["cron_key", "created_at"], name: "index_good_jobs_on_cron_key_and_created_at" - t.index ["cron_key", "cron_at"], name: "index_good_jobs_on_cron_key_and_cron_at", unique: true - t.index ["finished_at"], name: "index_good_jobs_jobs_on_finished_at", where: "((retried_good_job_id IS NULL) AND (finished_at IS NOT NULL))" - t.index ["priority", "created_at"], name: "index_good_jobs_jobs_on_priority_created_at_when_unfinished", order: { priority: "DESC NULLS LAST" }, where: "(finished_at IS NULL)" - t.index ["queue_name", "scheduled_at"], name: "index_good_jobs_on_queue_name_and_scheduled_at", where: "(finished_at IS NULL)" - t.index ["scheduled_at"], name: "index_good_jobs_on_scheduled_at", where: "(finished_at IS NULL)" - end - create_table "image_upload_transitions", force: :cascade do |t| t.string "to_state", null: false t.text "metadata" @@ -513,6 +446,17 @@ t.datetime "updated_at", precision: nil, null: false end + create_table "reference_types", force: :cascade do |t| + t.string "name" + t.string "reference_type" + t.string "reference_uri" + t.boolean "label", default: false + t.text "note" + t.integer "position" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "searches", force: :cascade do |t| t.text "query_params" t.integer "user_id" @@ -613,6 +557,7 @@ add_foreign_key "bulk_action_document_transitions", "bulk_action_documents" add_foreign_key "bulk_action_documents", "bulk_actions" add_foreign_key "bulk_action_transitions", "bulk_actions" + add_foreign_key "document_distributions", "reference_types" add_foreign_key "image_upload_transitions", "solr_document_sidecars" add_foreign_key "import_document_transitions", "import_documents" add_foreign_key "import_documents", "imports" diff --git a/db/seeds.rb b/db/seeds.rb index 53bbc16ca..5dc147044 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -5,4 +5,43 @@ # # cities = City.create([{ name: 'Chicago' }, { name: 'Copenhagen' }]) # Mayor.create(name: 'Emanuel', city: cities.first) -GeoblacklightAdmin::Engine.load_seed \ No newline at end of file +GeoblacklightAdmin::Engine.load_seed + +# db/seeds.rb + +reference_types = [ + { name: 'arcgis_dynamic_map_layer', reference_type: 'ArcGIS DynamicMapLayer', reference_uri: 'urn:x-esri:serviceType:ArcGIS#DynamicMapLayer', label: false, note: '-', position: 1 }, + { name: 'arcgis_feature_layer', reference_type: 'ArcGIS FeatureLayer', reference_uri: 'urn:x-esri:serviceType:ArcGIS#FeatureLayer', label: false, note: '-', position: 2 }, + { name: 'arcgis_image_map_layer', reference_type: 'ArcGIS ImageMapLayer', reference_uri: 'urn:x-esri:serviceType:ArcGIS#ImageMapLayer', label: false, note: '-', position: 3 }, + { name: 'arcgis_tiled_map_layer', reference_type: 'ArcGIS TiledMapLayer', reference_uri: 'urn:x-esri:serviceType:ArcGIS#TiledMapLayer', label: false, note: '-', position: 4 }, + { name: 'cog', reference_type: 'Cloud Optimized GeoTIFF (COG)', reference_uri: 'https://github.com/cogeotiff/cog-spec', label: false, note: '-', position: 5 }, + { name: 'documentation_download', reference_type: 'Data dictionary / supplemental documentation', reference_uri: 'http://lccn.loc.gov/sh85035852', label: false, note: 'Functions as a link to download documentation (not a viewer)', position: 6 }, + { name: 'documentation_external', reference_type: 'Documentation (External)', reference_uri: 'http://schema.org/url', label: false, note: '-', position: 7 }, + { name: 'download', reference_type: 'Download file', reference_uri: 'http://schema.org/downloadUrl', label: true, note: 'Link to download file (for multiple files see the multiple downloads guidelines)', position: 8 }, + { name: 'geo_json', reference_type: 'GeoJSON', reference_uri: 'http://geojson.org/geojson-spec.html', label: false, note: '-', position: 9 }, + { name: 'full_layer_description', reference_type: 'Full layer description', reference_uri: 'http://schema.org/url', label: false, note: 'To view further descriptive information about the layer or a link to its landing page', position: 10 }, + { name: 'iiif_image', reference_type: 'International Image Interoperability Framework (IIIF) Image API', reference_uri: 'http://iiif.io/api/image', label: false, note: 'Load the image viewer using Leaflet-IIIF', position: 11 }, + { name: 'iiif_manifest', reference_type: 'International Image Interoperability Framework (IIIF) Presentation API Manifest', reference_uri: 'http://iiif.io/api/presentation#manifest', label: false, note: 'View the IIIF manifest - uses the Clover viewer by default https://samvera-labs.github.io/clover-iiif/docs', position: 12 }, + { name: 'image', reference_type: 'Image file', reference_uri: 'http://schema.org/image', label: true, note: '-', position: 13 }, + { name: 'metadata_fgdc', reference_type: 'Metadata in FGDC', reference_uri: 'http://www.opengis.net/cat/csw/csdgm', label: false, note: 'Provides an HTML view of an XML file in the FGDC standard', position: 14 }, + { name: 'metadata_html', reference_type: 'Metadata in HTML', reference_uri: 'http://www.w3.org/1999/xhtml', label: false, note: 'View structured metadata in any standard expressed as HTML', position: 15 }, + { name: 'metadata_iso', reference_type: 'Metadata in ISO 19139', reference_uri: 'http://www.isotc211.org/schemas/2005/gmd/', label: false, note: 'Provides an HTML view of an XML file in the ISO 19139 standard', position: 16 }, + { name: 'metadata_mods', reference_type: 'Metadata in MODS', reference_uri: 'http://www.loc.gov/mods/v3', label: false, note: 'Provides a raw XML view of metadata in the MODS format', position: 17 }, + { name: 'oembed', reference_type: 'oEmbed', reference_uri: 'https://oembed.com', label: false, note: '-', position: 18 }, + { name: 'open_index_map', reference_type: 'OpenIndexMap', reference_uri: 'https://openindexmaps.org', label: false, note: 'Provides an interactive preview of a GeoJSON file formatted as an OpenIndexMap', position: 19 }, + { name: 'pmtiles', reference_type: 'PMTiles', reference_uri: 'https://github.com/protomaps/PMTiles', label: false, note: '-', position: 20 }, + { name: 'thumbnail', reference_type: 'Thumbnail file', reference_uri: 'http://schema.org/thumbnailUrl', label: true, note: '-', position: 21 }, + { name: 'tile_map_service', reference_type: 'Tile Mapping Service (TMS)', reference_uri: 'https://wiki.osgeo.org/wiki/Tile_Map_Service_Specification', label: false, note: '-', position: 22 }, + { name: 'tile_json', reference_type: 'TileJSON', reference_uri: 'https://github.com/mapbox/tilejson-spec', label: false, note: '-', position: 23 }, + { name: 'wcs', reference_type: 'Web Coverage Service (WCS)', reference_uri: 'http://www.opengis.net/def/serviceType/ogc/wcs', label: false, note: '-', position: 24 }, + { name: 'wfs', reference_type: 'Web Feature Service (WFS)', reference_uri: 'http://www.opengis.net/def/serviceType/ogc/wfs', label: false, note: 'Provides a to download generated vector datasets (GeoJSON, shapefile)', position: 25 }, + { name: 'wmts', reference_type: 'Web Mapping Service (WMS)', reference_uri: 'http://www.opengis.net/def/serviceType/ogc/wms', label: false, note: 'Provides a service to visually preview a layer and inspect its features', position: 26 }, + { name: 'wms', reference_type: 'WMTS', reference_uri: 'http://www.opengis.net/def/serviceType/ogc/wmts', label: false, note: '-', position: 27 }, + { name: 'xyz_tiles', reference_type: 'XYZ tiles', reference_uri: 'https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames', label: false, note: 'Link to an XYZ tile server', position: 28 } +] + +reference_types.each do |attributes| + ReferenceType.create!(attributes) +end + +puts "Reference types seeded successfully." \ No newline at end of file diff --git a/docs/geodev_references_audit_11072024.txt b/docs/geodev_references_audit_11072024.txt new file mode 100644 index 000000000..b29863268 --- /dev/null +++ b/docs/geodev_references_audit_11072024.txt @@ -0,0 +1,145 @@ +--- Audit Start --- + +NO MATCH — Duplicate holc-scan.jpg URL +Document: ed5705ef-7c1a-4497-b7af-fd971cee8e40 +CSV References Sorted: [["ed5705ef-7c1a-4497-b7af-fd971cee8e40", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=hudson-co.-nj", nil], ["ed5705ef-7c1a-4497-b7af-fd971cee8e40", "download", "https://s3.amazonaws.com/holc/tiles/NJ/HudsonCounty/1939/holc-scan.jpg", "GeoTIFF"], ["ed5705ef-7c1a-4497-b7af-fd971cee8e40", "download", "https://s3.amazonaws.com/holc/tiles/NJ/HudsonCounty/1939/holc-scan.jpg", "JPEG"]] +Document References Sorted: [["ed5705ef-7c1a-4497-b7af-fd971cee8e40", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=hudson-co.-nj", nil], ["ed5705ef-7c1a-4497-b7af-fd971cee8e40", "download", "https://s3.amazonaws.com/holc/tiles/NJ/HudsonCounty/1939/holc-scan.jpg", "JPEG"]] +Processed 1000 documents in this batch, total processed: 1000 +Processed 1000 documents in this batch, total processed: 2000 +Processed 1000 documents in this batch, total processed: 3000 + +NO MATCH - Duplicate rectified.zip URL +Document: 32027b5a-ea8a-42dd-875c-518cd3ab5811 +CSV References Sorted: [["32027b5a-ea8a-42dd-875c-518cd3ab5811", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=charleston-wv", nil], ["32027b5a-ea8a-42dd-875c-518cd3ab5811", "download", "https://s3.amazonaws.com/holc/tiles/WV/Charleston/1938/holc-scan.jpg", "JPG"], ["32027b5a-ea8a-42dd-875c-518cd3ab5811", "download", "https://s3.amazonaws.com/holc/tiles/WV/Charleston/1938/rectified.zip", "GeoTIFF"], ["32027b5a-ea8a-42dd-875c-518cd3ab5811", "download", "https://s3.amazonaws.com/holc/tiles/WV/Charleston/1938/rectified.zip", "GeoTIFF (inset)"]] +Document References Sorted: [["32027b5a-ea8a-42dd-875c-518cd3ab5811", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=charleston-wv", nil], ["32027b5a-ea8a-42dd-875c-518cd3ab5811", "download", "https://s3.amazonaws.com/holc/tiles/WV/Charleston/1938/holc-scan.jpg", "JPG"], ["32027b5a-ea8a-42dd-875c-518cd3ab5811", "download", "https://s3.amazonaws.com/holc/tiles/WV/Charleston/1938/rectified.zip", "GeoTIFF"]] +Processed 1000 documents in this batch, total processed: 4000 +Processed 1000 documents in this batch, total processed: 5000 +Processed 1000 documents in this batch, total processed: 6000 +Processed 1000 documents in this batch, total processed: 7000 +Processed 1000 documents in this batch, total processed: 8000 +Processed 1000 documents in this batch, total processed: 9000 +Processed 1000 documents in this batch, total processed: 10000 +Processed 1000 documents in this batch, total processed: 11000 +Processed 1000 documents in this batch, total processed: 12000 +Processed 1000 documents in this batch, total processed: 13000 +Processed 1000 documents in this batch, total processed: 14000 +Processed 1000 documents in this batch, total processed: 15000 +Processed 1000 documents in this batch, total processed: 16000 +Processed 1000 documents in this batch, total processed: 17000 +Processed 1000 documents in this batch, total processed: 18000 +Processed 1000 documents in this batch, total processed: 19000 +Processed 1000 documents in this batch, total processed: 20000 +Processed 1000 documents in this batch, total processed: 21000 +Processed 1000 documents in this batch, total processed: 22000 +Processed 1000 documents in this batch, total processed: 23000 +Processed 1000 documents in this batch, total processed: 24000 +Processed 1000 documents in this batch, total processed: 25000 +Processed 1000 documents in this batch, total processed: 26000 +Processed 1000 documents in this batch, total processed: 27000 +Processed 1000 documents in this batch, total processed: 28000 +Processed 1000 documents in this batch, total processed: 29000 +Processed 1000 documents in this batch, total processed: 30000 +Processed 1000 documents in this batch, total processed: 31000 +Processed 1000 documents in this batch, total processed: 32000 +Processed 1000 documents in this batch, total processed: 33000 +Processed 1000 documents in this batch, total processed: 34000 +Processed 1000 documents in this batch, total processed: 35000 +Processed 1000 documents in this batch, total processed: 36000 +Processed 1000 documents in this batch, total processed: 37000 +Processed 1000 documents in this batch, total processed: 38000 +Processed 1000 documents in this batch, total processed: 39000 +Processed 1000 documents in this batch, total processed: 40000 +Processed 1000 documents in this batch, total processed: 41000 +Processed 1000 documents in this batch, total processed: 42000 +Processed 1000 documents in this batch, total processed: 43000 +Processed 1000 documents in this batch, total processed: 44000 +Processed 1000 documents in this batch, total processed: 45000 +Processed 1000 documents in this batch, total processed: 46000 +Processed 1000 documents in this batch, total processed: 47000 +Processed 1000 documents in this batch, total processed: 48000 +Processed 1000 documents in this batch, total processed: 49000 +Processed 1000 documents in this batch, total processed: 50000 +Processed 1000 documents in this batch, total processed: 51000 +Processed 1000 documents in this batch, total processed: 52000 +Processed 1000 documents in this batch, total processed: 53000 +Processed 1000 documents in this batch, total processed: 54000 +Processed 1000 documents in this batch, total processed: 55000 +Processed 1000 documents in this batch, total processed: 56000 +Processed 1000 documents in this batch, total processed: 57000 +Processed 1000 documents in this batch, total processed: 58000 +Processed 1000 documents in this batch, total processed: 59000 +Processed 1000 documents in this batch, total processed: 60000 +Processed 1000 documents in this batch, total processed: 61000 +Processed 1000 documents in this batch, total processed: 62000 +Processed 1000 documents in this batch, total processed: 63000 +Processed 1000 documents in this batch, total processed: 64000 +Processed 1000 documents in this batch, total processed: 65000 +Processed 1000 documents in this batch, total processed: 66000 +Processed 1000 documents in this batch, total processed: 67000 +Processed 1000 documents in this batch, total processed: 68000 +Processed 1000 documents in this batch, total processed: 69000 +Processed 1000 documents in this batch, total processed: 70000 +Processed 1000 documents in this batch, total processed: 71000 +Processed 1000 documents in this batch, total processed: 72000 +Processed 1000 documents in this batch, total processed: 73000 +Processed 1000 documents in this batch, total processed: 74000 +Processed 1000 documents in this batch, total processed: 75000 +Processed 1000 documents in this batch, total processed: 76000 +Processed 1000 documents in this batch, total processed: 77000 +Processed 1000 documents in this batch, total processed: 78000 +Processed 1000 documents in this batch, total processed: 79000 +Processed 1000 documents in this batch, total processed: 80000 +Processed 1000 documents in this batch, total processed: 81000 +Processed 1000 documents in this batch, total processed: 82000 +Processed 1000 documents in this batch, total processed: 83000 +Processed 1000 documents in this batch, total processed: 84000 +Processed 1000 documents in this batch, total processed: 85000 +Processed 1000 documents in this batch, total processed: 86000 +Processed 1000 documents in this batch, total processed: 87000 +Processed 1000 documents in this batch, total processed: 88000 +Processed 1000 documents in this batch, total processed: 89000 +Processed 1000 documents in this batch, total processed: 90000 +Processed 1000 documents in this batch, total processed: 91000 +Processed 1000 documents in this batch, total processed: 92000 + +# ERROR - harvard_download in dct_references_s (we don't support those) +Error auditing references for document: harvard-g7064-s2-1834-k3 - # +Processed 1000 documents in this batch, total processed: 93000 +Processed 1000 documents in this batch, total processed: 94000 +Processed 1000 documents in this batch, total processed: 95000 +Processed 1000 documents in this batch, total processed: 96000 + +NO MATCH - Duplicate holc-scan.jpg URL +Document: 82a3b7a2-f1b8-4a64-9832-a227ddd1a66d +CSV References Sorted: [["82a3b7a2-f1b8-4a64-9832-a227ddd1a66d", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=staten-island-ny", nil], ["82a3b7a2-f1b8-4a64-9832-a227ddd1a66d", "download", "https://s3.amazonaws.com/holc/tiles/NY/StatenIsland/1938/holc-scan.jpg", "GeoTIFF"], ["82a3b7a2-f1b8-4a64-9832-a227ddd1a66d", "download", "https://s3.amazonaws.com/holc/tiles/NY/StatenIsland/1938/holc-scan.jpg", "JPEG"]] +Document References Sorted: [["82a3b7a2-f1b8-4a64-9832-a227ddd1a66d", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=staten-island-ny", nil], ["82a3b7a2-f1b8-4a64-9832-a227ddd1a66d", "download", "https://s3.amazonaws.com/holc/tiles/NY/StatenIsland/1938/holc-scan.jpg", "JPEG"]] +Processed 1000 documents in this batch, total processed: 97000 +Processed 1000 documents in this batch, total processed: 98000 +Processed 1000 documents in this batch, total processed: 99000 +Processed 1000 documents in this batch, total processed: 100000 +Processed 1000 documents in this batch, total processed: 101000 +Processed 1000 documents in this batch, total processed: 102000 +Processed 1000 documents in this batch, total processed: 103000 + +NO MATCH - Duplicate shp_bdry_votingdistricts.zip URL +Document: test +CSV References Sorted: [["test", "download", "https://example.com", "newLabel"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/kmz_bdry_votingdistricts.zip", "KMZ"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/kmz_bdry_votingdistricts.zip", "newLabel"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/shp_bdry_votingdistricts.zip", "Shapefile"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/shp_bdry_votingdistricts.zip", "anotherLabel"], ["test", "download", "https:/anotherExample.com", "anotherLabel"]] +Document References Sorted: [["test", "download", "https://example.com", "newLabel"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/kmz_bdry_votingdistricts.zip", "KMZ"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/shp_bdry_votingdistricts.zip", "Shapefile"], ["test", "download", "https:/anotherExample.com", "anotherLabel"]] + +NO MATCH - Bad COG (array of URLs) +Document: btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166 +CSV References Sorted: [["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "cog", ["https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/51011e42-c559-47dd-8e18-f6388ecdc083/f2d92d1dfb83d2361547c62c5321be52.tif", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/6b7500e0-acea-45c3-a8b2-47a8b2a5fb20/acc4aa6b5e9ff7f346a6a3f1b13cc2df.tif"], nil], ["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "documentation_external", "http://maps.indiana.edu/layerGallery.html", nil], ["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "download", "http://maps.indiana.edu/download/Environment/Land_Cover_Impervious_Surfaces_2006.zip", "GeoTIFF"]] +Document References Sorted: [["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "cog", "[\"https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/51011e42-c559-47dd-8e18-f6388ecdc083/f2d92d1dfb83d2361547c62c5321be52.tif\", \"https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/6b7500e0-acea-45c3-a8b2-47a8b2a5fb20/acc4aa6b5e9ff7f346a6a3f1b13cc2df.tif\"]", nil], ["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "documentation_external", "http://maps.indiana.edu/layerGallery.html", nil], ["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "download", "http://maps.indiana.edu/download/Environment/Land_Cover_Impervious_Surfaces_2006.zip", "GeoTIFF"]] +Processed 1000 documents in this batch, total processed: 104000 +Processed 1000 documents in this batch, total processed: 105000 +Processed 1000 documents in this batch, total processed: 106000 +Processed 1000 documents in this batch, total processed: 107000 +Processed 1000 documents in this batch, total processed: 108000 + +NO MATCH - Bad image (array of URLs) +Document: 59bd8ed8-088c-4932-97c9-544135dd9d64 +CSV References Sorted: [["59bd8ed8-088c-4932-97c9-544135dd9d64", "documentation_external", "https://michiganology.org/uncategorized/IO_59bd8ed8-088c-4932-97c9-544135dd9d64", nil], ["59bd8ed8-088c-4932-97c9-544135dd9d64", "image", ["https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/88b43100-06a1-47e0-8fe0-797b1768f4e0/952fffe1b6f9f9367fdec0cfca0b3c2b.png", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/29421b81-8152-4fff-972e-90f98ed57ee2/aa172e2dd422e038b09b0f081e82084e.png"], nil]] +Document References Sorted: [["59bd8ed8-088c-4932-97c9-544135dd9d64", "documentation_external", "https://michiganology.org/uncategorized/IO_59bd8ed8-088c-4932-97c9-544135dd9d64", nil], ["59bd8ed8-088c-4932-97c9-544135dd9d64", "image", "[\"https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/88b43100-06a1-47e0-8fe0-797b1768f4e0/952fffe1b6f9f9367fdec0cfca0b3c2b.png\", \"https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/29421b81-8152-4fff-972e-90f98ed57ee2/aa172e2dd422e038b09b0f081e82084e.png\"]", nil]] +Processed 1000 documents in this batch, total processed: 109000 +Processed 178 documents in this batch, total processed: 109178 +--- Audit End --- diff --git a/docs/localhost_references_audit_11082024.txt b/docs/localhost_references_audit_11082024.txt new file mode 100644 index 000000000..a399aaf20 --- /dev/null +++ b/docs/localhost_references_audit_11082024.txt @@ -0,0 +1,229 @@ +--- Audit Start --- + +NO MATCH - Duplicate holc-scan.jpg URL +Document: ed5705ef-7c1a-4497-b7af-fd971cee8e40 +CSV References Sorted: [["ed5705ef-7c1a-4497-b7af-fd971cee8e40", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=hudson-co.-nj", nil], ["ed5705ef-7c1a-4497-b7af-fd971cee8e40", "download", "https://s3.amazonaws.com/holc/tiles/NJ/HudsonCounty/1939/holc-scan.jpg", "GeoTIFF"], ["ed5705ef-7c1a-4497-b7af-fd971cee8e40", "download", "https://s3.amazonaws.com/holc/tiles/NJ/HudsonCounty/1939/holc-scan.jpg", "JPEG"]] +Document References Sorted: [["ed5705ef-7c1a-4497-b7af-fd971cee8e40", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=hudson-co.-nj", nil], ["ed5705ef-7c1a-4497-b7af-fd971cee8e40", "download", "https://s3.amazonaws.com/holc/tiles/NJ/HudsonCounty/1939/holc-scan.jpg", "JPEG"]] +Processed 1000 documents in this batch, total processed: 1000 +Processed 1000 documents in this batch, total processed: 2000 +Processed 1000 documents in this batch, total processed: 3000 + +NO MATCH - Duplicate rectified.zip URL +Document: 32027b5a-ea8a-42dd-875c-518cd3ab5811 +CSV References Sorted: [["32027b5a-ea8a-42dd-875c-518cd3ab5811", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=charleston-wv", nil], ["32027b5a-ea8a-42dd-875c-518cd3ab5811", "download", "https://s3.amazonaws.com/holc/tiles/WV/Charleston/1938/holc-scan.jpg", "JPG"], ["32027b5a-ea8a-42dd-875c-518cd3ab5811", "download", "https://s3.amazonaws.com/holc/tiles/WV/Charleston/1938/rectified.zip", "GeoTIFF"], ["32027b5a-ea8a-42dd-875c-518cd3ab5811", "download", "https://s3.amazonaws.com/holc/tiles/WV/Charleston/1938/rectified.zip", "GeoTIFF (inset)"]] +Document References Sorted: [["32027b5a-ea8a-42dd-875c-518cd3ab5811", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=charleston-wv", nil], ["32027b5a-ea8a-42dd-875c-518cd3ab5811", "download", "https://s3.amazonaws.com/holc/tiles/WV/Charleston/1938/holc-scan.jpg", "JPG"], ["32027b5a-ea8a-42dd-875c-518cd3ab5811", "download", "https://s3.amazonaws.com/holc/tiles/WV/Charleston/1938/rectified.zip", "GeoTIFF"]] +Processed 1000 documents in this batch, total processed: 4000 + +NO MATCH +Document: 8cd3d4ba-75b1-4cbe-b978-fa23f5fbdca7 +CSV References Sorted: [["8cd3d4ba-75b1-4cbe-b978-fa23f5fbdca7", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["8cd3d4ba-75b1-4cbe-b978-fa23f5fbdca7", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/bcddb6b0-5a7c-4337-b5b2-385c03696b14/dba7e29239b5d738ebc61556ced3bd3f.zip", "wicolu02.zip"], ["8cd3d4ba-75b1-4cbe-b978-fa23f5fbdca7", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/bcddb6b0-5a7c-4337-b5b2-385c03696b14/dba7e29239b5d738ebc61556ced3bd3f.zip", "Shapefile"], ["8cd3d4ba-75b1-4cbe-b978-fa23f5fbdca7", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/83f2dd82-d3be-4064-85e1-fe0b3171e1fc/ba285518fe6a561b627dbdda66ba4455.pmtiles", nil]] +Document References Sorted: [["8cd3d4ba-75b1-4cbe-b978-fa23f5fbdca7", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["8cd3d4ba-75b1-4cbe-b978-fa23f5fbdca7", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/bcddb6b0-5a7c-4337-b5b2-385c03696b14/dba7e29239b5d738ebc61556ced3bd3f.zip", "wicolu02.zip"], ["8cd3d4ba-75b1-4cbe-b978-fa23f5fbdca7", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/83f2dd82-d3be-4064-85e1-fe0b3171e1fc/ba285518fe6a561b627dbdda66ba4455.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 5000 +Processed 1000 documents in this batch, total processed: 6000 +Processed 1000 documents in this batch, total processed: 7000 +Processed 1000 documents in this batch, total processed: 8000 +Processed 1000 documents in this batch, total processed: 9000 + +NO MATCH +Document: 9e5aa5c8-071f-43ab-9121-19433ca85d6e +CSV References Sorted: [["9e5aa5c8-071f-43ab-9121-19433ca85d6e", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["9e5aa5c8-071f-43ab-9121-19433ca85d6e", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/7c39c703-92d2-43f2-bd70-31ee228ff6ae/ac773faa312d4889104cf69d5f2a01e5.zip", "queelu02.zip"], ["9e5aa5c8-071f-43ab-9121-19433ca85d6e", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/7c39c703-92d2-43f2-bd70-31ee228ff6ae/ac773faa312d4889104cf69d5f2a01e5.zip", "Shapefile"], ["9e5aa5c8-071f-43ab-9121-19433ca85d6e", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/66eacb4b-9288-43dc-92aa-edee0c79cac0/2b2007b7b67d73828c76e80f17bb5a08.pmtiles", nil]] +Document References Sorted: [["9e5aa5c8-071f-43ab-9121-19433ca85d6e", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["9e5aa5c8-071f-43ab-9121-19433ca85d6e", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/7c39c703-92d2-43f2-bd70-31ee228ff6ae/ac773faa312d4889104cf69d5f2a01e5.zip", "queelu02.zip"], ["9e5aa5c8-071f-43ab-9121-19433ca85d6e", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/66eacb4b-9288-43dc-92aa-edee0c79cac0/2b2007b7b67d73828c76e80f17bb5a08.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 10000 + +NO MATCH +Document: btaa-0d1f5c0e-f93f-4889-be34-16104f9c6927 +CSV References Sorted: [["btaa-0d1f5c0e-f93f-4889-be34-16104f9c6927", "cog", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/ad2ac386-d3bf-4830-8044-9a1118805f09/7f3333d450a445df7fa068d3aee9537a.tif", nil], ["btaa-0d1f5c0e-f93f-4889-be34-16104f9c6927", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/35843439-cf4c-485d-8176-581d9cf12305/c62a6e98889841135d3b3b611833f0ed.zip", "Aquifer_Sensitivity_Near_Surface.zip"], ["btaa-0d1f5c0e-f93f-4889-be34-16104f9c6927", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/35843439-cf4c-485d-8176-581d9cf12305/c62a6e98889841135d3b3b611833f0ed.zip", "GeoTIFF"]] +Document References Sorted: [["btaa-0d1f5c0e-f93f-4889-be34-16104f9c6927", "cog", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/ad2ac386-d3bf-4830-8044-9a1118805f09/7f3333d450a445df7fa068d3aee9537a.tif", nil], ["btaa-0d1f5c0e-f93f-4889-be34-16104f9c6927", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/35843439-cf4c-485d-8176-581d9cf12305/c62a6e98889841135d3b3b611833f0ed.zip", "Aquifer_Sensitivity_Near_Surface.zip"]] +Processed 1000 documents in this batch, total processed: 11000 + +NO MATCH +Document: 0f9688d8-de7d-4d7c-8339-9c2ac314ee98 +CSV References Sorted: [["0f9688d8-de7d-4d7c-8339-9c2ac314ee98", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["0f9688d8-de7d-4d7c-8339-9c2ac314ee98", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/155cac25-9e6e-4042-905b-8ea59aa3fa64/2c58fb8d4806ef2b64312c64a9817032.zip", "prinlu02.zip"], ["0f9688d8-de7d-4d7c-8339-9c2ac314ee98", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/155cac25-9e6e-4042-905b-8ea59aa3fa64/2c58fb8d4806ef2b64312c64a9817032.zip", "Shapefile"], ["0f9688d8-de7d-4d7c-8339-9c2ac314ee98", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/ea6a508b-36fa-4f5a-9794-203f78379871/4b5e2ffea0cdebd9240693757ad6cf7f.pmtiles", nil]] +Document References Sorted: [["0f9688d8-de7d-4d7c-8339-9c2ac314ee98", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["0f9688d8-de7d-4d7c-8339-9c2ac314ee98", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/155cac25-9e6e-4042-905b-8ea59aa3fa64/2c58fb8d4806ef2b64312c64a9817032.zip", "prinlu02.zip"], ["0f9688d8-de7d-4d7c-8339-9c2ac314ee98", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/ea6a508b-36fa-4f5a-9794-203f78379871/4b5e2ffea0cdebd9240693757ad6cf7f.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 12000 + +NO MATCH +Document: 5fc8dc18-62e6-4316-ba99-1a61534ea088 +CSV References Sorted: [["5fc8dc18-62e6-4316-ba99-1a61534ea088", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["5fc8dc18-62e6-4316-ba99-1a61534ea088", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/2e1ba025-123b-4eaf-8a0c-ef3d5581ac16/a7b1ffe10d7fffd5616c5ba90a0b97cc.zip", "washlu02.zip"], ["5fc8dc18-62e6-4316-ba99-1a61534ea088", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/2e1ba025-123b-4eaf-8a0c-ef3d5581ac16/a7b1ffe10d7fffd5616c5ba90a0b97cc.zip", "Shapefile"], ["5fc8dc18-62e6-4316-ba99-1a61534ea088", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/ec0fdd6d-1646-44b3-88f9-223e341adfc4/5670f2caa5330fd6a1579d5b55350dda.pmtiles", nil]] +Document References Sorted: [["5fc8dc18-62e6-4316-ba99-1a61534ea088", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["5fc8dc18-62e6-4316-ba99-1a61534ea088", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/2e1ba025-123b-4eaf-8a0c-ef3d5581ac16/a7b1ffe10d7fffd5616c5ba90a0b97cc.zip", "washlu02.zip"], ["5fc8dc18-62e6-4316-ba99-1a61534ea088", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/ec0fdd6d-1646-44b3-88f9-223e341adfc4/5670f2caa5330fd6a1579d5b55350dda.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 13000 +Processed 1000 documents in this batch, total processed: 14000 +Processed 1000 documents in this batch, total processed: 15000 +Processed 1000 documents in this batch, total processed: 16000 + +NO MATCH +Document: e65d1fb2-5e3f-47b5-9341-d638931b508c +CSV References Sorted: [["e65d1fb2-5e3f-47b5-9341-d638931b508c", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["e65d1fb2-5e3f-47b5-9341-d638931b508c", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/742a7bce-eeb2-43ec-9f34-aff92ff38137/6287bb36020c79d998db621e8a374d6a.zip", "talblu02.zip"], ["e65d1fb2-5e3f-47b5-9341-d638931b508c", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/742a7bce-eeb2-43ec-9f34-aff92ff38137/6287bb36020c79d998db621e8a374d6a.zip", "Shapefile"], ["e65d1fb2-5e3f-47b5-9341-d638931b508c", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/f9580be1-dd0c-4d2a-a81b-2d33fcf3308f/c37de751689c4351d44699d4158ea205.pmtiles", nil]] +Document References Sorted: [["e65d1fb2-5e3f-47b5-9341-d638931b508c", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["e65d1fb2-5e3f-47b5-9341-d638931b508c", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/742a7bce-eeb2-43ec-9f34-aff92ff38137/6287bb36020c79d998db621e8a374d6a.zip", "talblu02.zip"], ["e65d1fb2-5e3f-47b5-9341-d638931b508c", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/f9580be1-dd0c-4d2a-a81b-2d33fcf3308f/c37de751689c4351d44699d4158ea205.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 17000 + +NO MATCH +Document: 099bda09-d518-4a8e-8981-c0eb6e48495c +CSV References Sorted: [["099bda09-d518-4a8e-8981-c0eb6e48495c", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["099bda09-d518-4a8e-8981-c0eb6e48495c", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/cea57932-72aa-4721-b94a-cec77d264b8f/8c32420230a07bbfab0bff0303c0bc99.zip", "howalu02.zip"], ["099bda09-d518-4a8e-8981-c0eb6e48495c", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/cea57932-72aa-4721-b94a-cec77d264b8f/8c32420230a07bbfab0bff0303c0bc99.zip", "Shapefile"], ["099bda09-d518-4a8e-8981-c0eb6e48495c", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/fec5b374-bed8-4397-9e7b-2533ea916031/2900232d9f00f80bf38fa7db455fdc7c.pmtiles", nil]] +Document References Sorted: [["099bda09-d518-4a8e-8981-c0eb6e48495c", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["099bda09-d518-4a8e-8981-c0eb6e48495c", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/cea57932-72aa-4721-b94a-cec77d264b8f/8c32420230a07bbfab0bff0303c0bc99.zip", "howalu02.zip"], ["099bda09-d518-4a8e-8981-c0eb6e48495c", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/fec5b374-bed8-4397-9e7b-2533ea916031/2900232d9f00f80bf38fa7db455fdc7c.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 18000 + +NO MATCH +Document: demo-01 +CSV References Sorted: [["demo-01", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/1386180e-8d9b-4adc-b00f-d4fe1ce1d36c/448c534106e7d70f12f72e357351290c.zip", "geodatabase.gdb.zip"], ["demo-01", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/8f6f9903-fa07-40b8-b6bb-85316081c020/32332dc98388b849f8cbacbe9d706818.zip", "shapefile.zip"], ["demo-01", "metadata_html", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/9110e3ff-e5ca-4fd0-91a4-97417c2cdce8/57cf879488900529fea70d7654ccd0b9.html", nil], ["demo-01", "metadata_iso", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/3dab566a-9d04-40bd-b1fb-083a6e8bede1/fe994aa784d30437a134eb80ecf6212d.xml", nil], ["demo-01", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/f78f027e-5ab2-4db4-a2a1-c34f9fe36626/495373519c7039d18be25db5bfa5e9a0.pmtiles", nil]] +Document References Sorted: [["demo-01", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/1386180e-8d9b-4adc-b00f-d4fe1ce1d36c/448c534106e7d70f12f72e357351290c.zip", "geodatabase.gdb.zip"], ["demo-01", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/8f6f9903-fa07-40b8-b6bb-85316081c020/32332dc98388b849f8cbacbe9d706818.zip", "shapefile.zip"], ["demo-01", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/1386180e-8d9b-4adc-b00f-d4fe1ce1d36c/448c534106e7d70f12f72e357351290c.zip", "geodatabase.gdb.zip"], ["demo-01", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/8f6f9903-fa07-40b8-b6bb-85316081c020/32332dc98388b849f8cbacbe9d706818.zip", "shapefile.zip"], ["demo-01", "metadata_html", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/9110e3ff-e5ca-4fd0-91a4-97417c2cdce8/57cf879488900529fea70d7654ccd0b9.html", nil], ["demo-01", "metadata_iso", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/3dab566a-9d04-40bd-b1fb-083a6e8bede1/fe994aa784d30437a134eb80ecf6212d.xml", nil], ["demo-01", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/f78f027e-5ab2-4db4-a2a1-c34f9fe36626/495373519c7039d18be25db5bfa5e9a0.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 19000 +Processed 1000 documents in this batch, total processed: 20000 + +NO MATCH +Document: e0d15cad-86d4-4509-9b4e-758c1100d38d +CSV References Sorted: [["e0d15cad-86d4-4509-9b4e-758c1100d38d", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["e0d15cad-86d4-4509-9b4e-758c1100d38d", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/ece861d7-0128-4181-9cd8-9e675d5bff28/b1a24a50f4c4a7a54dce428a5e7de09f.zip", "somelu02.zip"], ["e0d15cad-86d4-4509-9b4e-758c1100d38d", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/ece861d7-0128-4181-9cd8-9e675d5bff28/b1a24a50f4c4a7a54dce428a5e7de09f.zip", "Shapefile"], ["e0d15cad-86d4-4509-9b4e-758c1100d38d", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/5e0b4359-2466-4c9a-8a67-e850373c92da/9016f58c38a31109f989a47094a75c35.pmtiles", nil]] +Document References Sorted: [["e0d15cad-86d4-4509-9b4e-758c1100d38d", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["e0d15cad-86d4-4509-9b4e-758c1100d38d", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/ece861d7-0128-4181-9cd8-9e675d5bff28/b1a24a50f4c4a7a54dce428a5e7de09f.zip", "somelu02.zip"], ["e0d15cad-86d4-4509-9b4e-758c1100d38d", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/5e0b4359-2466-4c9a-8a67-e850373c92da/9016f58c38a31109f989a47094a75c35.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 21000 +Processed 1000 documents in this batch, total processed: 22000 +Processed 1000 documents in this batch, total processed: 23000 +Processed 1000 documents in this batch, total processed: 24000 +Processed 1000 documents in this batch, total processed: 25000 +Processed 1000 documents in this batch, total processed: 26000 + +NO MATCH +Document: f245136b-0d60-4315-abb0-52d0fbee4c5c +CSV References Sorted: [["f245136b-0d60-4315-abb0-52d0fbee4c5c", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["f245136b-0d60-4315-abb0-52d0fbee4c5c", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/fb544a75-16c4-4d84-88f3-7a3a26a330a4/14c781373ed8c16fa3ee69a6eed2c10b.zip", "charlu02.zip"], ["f245136b-0d60-4315-abb0-52d0fbee4c5c", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/fb544a75-16c4-4d84-88f3-7a3a26a330a4/14c781373ed8c16fa3ee69a6eed2c10b.zip", "Shapefile"], ["f245136b-0d60-4315-abb0-52d0fbee4c5c", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/d3c052e8-2021-4010-9f9b-28cd39ced401/7b9a5f03ec43669e51b1c38c9a40df4a.pmtiles", nil]] +Document References Sorted: [["f245136b-0d60-4315-abb0-52d0fbee4c5c", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["f245136b-0d60-4315-abb0-52d0fbee4c5c", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/fb544a75-16c4-4d84-88f3-7a3a26a330a4/14c781373ed8c16fa3ee69a6eed2c10b.zip", "charlu02.zip"], ["f245136b-0d60-4315-abb0-52d0fbee4c5c", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/d3c052e8-2021-4010-9f9b-28cd39ced401/7b9a5f03ec43669e51b1c38c9a40df4a.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 27000 +Processed 1000 documents in this batch, total processed: 28000 +Processed 1000 documents in this batch, total processed: 29000 +Processed 1000 documents in this batch, total processed: 30000 +Processed 1000 documents in this batch, total processed: 31000 + +NO MATCH +Document: aug-5-2024 +CSV References Sorted: [["aug-5-2024", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/cc88fc5e-9dc4-4f09-89cf-22335910716c/91b4c31ee446c420390a16571a3971ab.csv", nil], ["aug-5-2024", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/02c06377-071d-4019-b89b-ee5b937776cd/efd61e3342b3b14b389ba7ad08d5065e.zip", "Geodatabase"], ["aug-5-2024", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/e0984e99-e174-45ac-be0e-a17101197f8e/7dc9b32d545b515323d5ae068aee9026.zip", "Shapefile"], ["aug-5-2024", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/e0984e99-e174-45ac-be0e-a17101197f8e/7dc9b32d545b515323d5ae068aee9026.zip", "Mixed"], ["aug-5-2024", "metadata_fgdc", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/4cb38259-b1ac-4ab7-b075-dc1cd548ba7b/3cec5aeb7d6f68a0939204f2d3c3993c.xml", nil], ["aug-5-2024", "metadata_html", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/ba4716ea-c86d-4ea1-98b5-a7433daacdfb/dc523dc6e0a1d6533b8ce49fcee297e4.html", nil], ["aug-5-2024", "metadata_iso", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/b5c33f20-16a5-4bad-83e1-8cbc65aad67a/2fbc5ed3ad78410cbf7c1d39ac7470bb.xml", nil], ["aug-5-2024", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/af264c4c-862a-46dd-a3b4-88973a8fa754/c8858797b551c94a92ecbe8387d37b85.pmtiles", nil]] +Document References Sorted: [["aug-5-2024", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/cc88fc5e-9dc4-4f09-89cf-22335910716c/91b4c31ee446c420390a16571a3971ab.csv", nil], ["aug-5-2024", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/02c06377-071d-4019-b89b-ee5b937776cd/efd61e3342b3b14b389ba7ad08d5065e.zip", "Geodatabase"], ["aug-5-2024", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/e0984e99-e174-45ac-be0e-a17101197f8e/7dc9b32d545b515323d5ae068aee9026.zip", "Shapefile"], ["aug-5-2024", "metadata_fgdc", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/4cb38259-b1ac-4ab7-b075-dc1cd548ba7b/3cec5aeb7d6f68a0939204f2d3c3993c.xml", nil], ["aug-5-2024", "metadata_html", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/ba4716ea-c86d-4ea1-98b5-a7433daacdfb/dc523dc6e0a1d6533b8ce49fcee297e4.html", nil], ["aug-5-2024", "metadata_iso", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/b5c33f20-16a5-4bad-83e1-8cbc65aad67a/2fbc5ed3ad78410cbf7c1d39ac7470bb.xml", nil], ["aug-5-2024", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/af264c4c-862a-46dd-a3b4-88973a8fa754/c8858797b551c94a92ecbe8387d37b85.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 32000 + +NO MATCH +Document: btaa-3020c203-5325-4bec-92e3-4a16e267c14e +CSV References Sorted: [["btaa-3020c203-5325-4bec-92e3-4a16e267c14e", "documentation_external", "http://maps.indiana.edu/previewMaps/Infrastructure/Railroads_Active_Abandoned_INDOT.html", nil], ["btaa-3020c203-5325-4bec-92e3-4a16e267c14e", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/0027dc65-5d68-4d1e-a996-efd7886a1f5c/360dbb031fbaa6f01db6e6046beb228b.zip", "Shapefile"]] +Document References Sorted: [["btaa-3020c203-5325-4bec-92e3-4a16e267c14e", "documentation_external", "http://maps.indiana.edu/previewMaps/Infrastructure/Railroads_Active_Abandoned_INDOT.html", nil], ["btaa-3020c203-5325-4bec-92e3-4a16e267c14e", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/0027dc65-5d68-4d1e-a996-efd7886a1f5c/360dbb031fbaa6f01db6e6046beb228b.zip", "Shapefile"], ["btaa-3020c203-5325-4bec-92e3-4a16e267c14e", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/0027dc65-5d68-4d1e-a996-efd7886a1f5c/360dbb031fbaa6f01db6e6046beb228b.zip", "Shapefile"]] +Processed 1000 documents in this batch, total processed: 33000 +Processed 1000 documents in this batch, total processed: 34000 + +NO MATCH +Document: 87d2a9e8-7c16-40a9-8fab-155a1c6b209d +CSV References Sorted: [["87d2a9e8-7c16-40a9-8fab-155a1c6b209d", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["87d2a9e8-7c16-40a9-8fab-155a1c6b209d", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/bcec399e-2730-49de-99ce-863ea2153c66/5923ebeafb0f6cd35886c0055b8a9f09.zip", "calvlu02.zip"], ["87d2a9e8-7c16-40a9-8fab-155a1c6b209d", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/bcec399e-2730-49de-99ce-863ea2153c66/5923ebeafb0f6cd35886c0055b8a9f09.zip", "Shapefile"], ["87d2a9e8-7c16-40a9-8fab-155a1c6b209d", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/6bf2d6a1-3027-4a44-ab14-ce9aa6e24565/d0460a41554de8fe92e78393338423c7.pmtiles", nil]] +Document References Sorted: [["87d2a9e8-7c16-40a9-8fab-155a1c6b209d", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["87d2a9e8-7c16-40a9-8fab-155a1c6b209d", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/bcec399e-2730-49de-99ce-863ea2153c66/5923ebeafb0f6cd35886c0055b8a9f09.zip", "calvlu02.zip"], ["87d2a9e8-7c16-40a9-8fab-155a1c6b209d", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/6bf2d6a1-3027-4a44-ab14-ce9aa6e24565/d0460a41554de8fe92e78393338423c7.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 35000 +Processed 1000 documents in this batch, total processed: 36000 +Processed 1000 documents in this batch, total processed: 37000 + +NO MATCH +Document: df107945-d91f-40f0-8802-504544918d48 +CSV References Sorted: [["df107945-d91f-40f0-8802-504544918d48", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["df107945-d91f-40f0-8802-504544918d48", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/520d12e8-a4f3-426a-aa51-4c9f61f7448e/4360868ca1fa192b55fe04ce71d0882d.zip", "fredlu02.zip"], ["df107945-d91f-40f0-8802-504544918d48", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/520d12e8-a4f3-426a-aa51-4c9f61f7448e/4360868ca1fa192b55fe04ce71d0882d.zip", "Shapefile"], ["df107945-d91f-40f0-8802-504544918d48", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/a0543f18-79a1-49d3-ba9a-dda899ca9574/61d5276991895701d076ace7d6881e8d.pmtiles", nil]] +Document References Sorted: [["df107945-d91f-40f0-8802-504544918d48", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["df107945-d91f-40f0-8802-504544918d48", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/520d12e8-a4f3-426a-aa51-4c9f61f7448e/4360868ca1fa192b55fe04ce71d0882d.zip", "fredlu02.zip"], ["df107945-d91f-40f0-8802-504544918d48", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/a0543f18-79a1-49d3-ba9a-dda899ca9574/61d5276991895701d076ace7d6881e8d.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 38000 + +NO MATCH +Document: ca1590a9-6077-4e2c-a029-beefc614c9a2 +CSV References Sorted: [["ca1590a9-6077-4e2c-a029-beefc614c9a2", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["ca1590a9-6077-4e2c-a029-beefc614c9a2", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/db90a215-2255-41d2-9f75-ad12c3515257/600fda0a7b49192369ac0262fde3610d.zip", "harflu02.zip"], ["ca1590a9-6077-4e2c-a029-beefc614c9a2", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/db90a215-2255-41d2-9f75-ad12c3515257/600fda0a7b49192369ac0262fde3610d.zip", "Shapefile"], ["ca1590a9-6077-4e2c-a029-beefc614c9a2", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/12a76d2b-eb87-4f81-a135-f5cecf087304/b5d74dcbaa8216fb199a9e55f2e4d555.pmtiles", nil]] +Document References Sorted: [["ca1590a9-6077-4e2c-a029-beefc614c9a2", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["ca1590a9-6077-4e2c-a029-beefc614c9a2", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/db90a215-2255-41d2-9f75-ad12c3515257/600fda0a7b49192369ac0262fde3610d.zip", "harflu02.zip"], ["ca1590a9-6077-4e2c-a029-beefc614c9a2", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/12a76d2b-eb87-4f81-a135-f5cecf087304/b5d74dcbaa8216fb199a9e55f2e4d555.pmtiles", nil]] +Processed 1000 documents in this batch, total processed: 39000 +Processed 1000 documents in this batch, total processed: 40000 +Processed 1000 documents in this batch, total processed: 41000 + +NO MATCH +Document: a3b32ab0-3149-4f1d-b6e9-9c51dafe5613 +CSV References Sorted: [["a3b32ab0-3149-4f1d-b6e9-9c51dafe5613", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["a3b32ab0-3149-4f1d-b6e9-9c51dafe5613", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/eaa41daf-752a-4f4f-9c43-1742f3e8f2e5/ce0ece3a00dd056344c18f11010d086d.zip", "garrlu02.zip"], ["a3b32ab0-3149-4f1d-b6e9-9c51dafe5613", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/eaa41daf-752a-4f4f-9c43-1742f3e8f2e5/ce0ece3a00dd056344c18f11010d086d.zip", "Shapefile"], ["a3b32ab0-3149-4f1d-b6e9-9c51dafe5613", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/46ed8c47-6729-4d5c-99a2-c31e91207bbf/daa05ab70dbbadd384996454a1fa92a1.pmtiles", nil]] +Document References Sorted: [["a3b32ab0-3149-4f1d-b6e9-9c51dafe5613", "documentation_download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/28f57e52-85c8-4797-b80d-658e6bb3e36e/2ace10f5adf37c667edfd39ff69a80f9.html", nil], ["a3b32ab0-3149-4f1d-b6e9-9c51dafe5613", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/eaa41daf-752a-4f4f-9c43-1742f3e8f2e5/ce0ece3a00dd056344c18f11010d086d.zip", "garrlu02.zip"], ["a3b32ab0-3149-4f1d-b6e9-9c51dafe5613", "pmtiles", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/46ed8c47-6729-4d5c-99a2-c31e91207bbf/daa05ab70dbbadd384996454a1fa92a1.pmtiles", nil]] + +NO MATCH +Document: 1da88d05-7b51-48ae-b5d5-4711c052b3f2 +CSV References Sorted: [["1da88d05-7b51-48ae-b5d5-4711c052b3f2", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/f55e0cc3-3684-491c-99bc-702014c2b61b/293cd53d936793143105427e9a2b6781.zip", "Surficial_Unconsolidated_Thickness_DEM.zip"]] +Document References Sorted: [["1da88d05-7b51-48ae-b5d5-4711c052b3f2", "download", "https://eric-geoportal-test.s3.amazonaws.com/store/asset/f55e0cc3-3684-491c-99bc-702014c2b61b/293cd53d936793143105427e9a2b6781.zip", "Surficial_Unconsolidated_Thickness_DEM.zip"], ["1da88d05-7b51-48ae-b5d5-4711c052b3f2", "download", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/f55e0cc3-3684-491c-99bc-702014c2b61b/293cd53d936793143105427e9a2b6781.zip", "Surficial_Unconsolidated_Thickness_DEM.zip"]] +Processed 1000 documents in this batch, total processed: 42000 +Processed 1000 documents in this batch, total processed: 43000 +Processed 1000 documents in this batch, total processed: 44000 +Processed 1000 documents in this batch, total processed: 45000 +Processed 1000 documents in this batch, total processed: 46000 +Processed 1000 documents in this batch, total processed: 47000 +Processed 1000 documents in this batch, total processed: 48000 +Processed 1000 documents in this batch, total processed: 49000 +Processed 1000 documents in this batch, total processed: 50000 +Processed 1000 documents in this batch, total processed: 51000 +Processed 1000 documents in this batch, total processed: 52000 +Processed 1000 documents in this batch, total processed: 53000 +Processed 1000 documents in this batch, total processed: 54000 +Processed 1000 documents in this batch, total processed: 55000 +Processed 1000 documents in this batch, total processed: 56000 +Processed 1000 documents in this batch, total processed: 57000 +Processed 1000 documents in this batch, total processed: 58000 +Processed 1000 documents in this batch, total processed: 59000 +Processed 1000 documents in this batch, total processed: 60000 +Processed 1000 documents in this batch, total processed: 61000 +Processed 1000 documents in this batch, total processed: 62000 +Processed 1000 documents in this batch, total processed: 63000 +Processed 1000 documents in this batch, total processed: 64000 +Processed 1000 documents in this batch, total processed: 65000 +Processed 1000 documents in this batch, total processed: 66000 +Processed 1000 documents in this batch, total processed: 67000 +Processed 1000 documents in this batch, total processed: 68000 +Processed 1000 documents in this batch, total processed: 69000 +Processed 1000 documents in this batch, total processed: 70000 +Processed 1000 documents in this batch, total processed: 71000 +Processed 1000 documents in this batch, total processed: 72000 +Processed 1000 documents in this batch, total processed: 73000 +Processed 1000 documents in this batch, total processed: 74000 +Processed 1000 documents in this batch, total processed: 75000 +Processed 1000 documents in this batch, total processed: 76000 +Processed 1000 documents in this batch, total processed: 77000 +Processed 1000 documents in this batch, total processed: 78000 +Processed 1000 documents in this batch, total processed: 79000 +Processed 1000 documents in this batch, total processed: 80000 +Processed 1000 documents in this batch, total processed: 81000 +Processed 1000 documents in this batch, total processed: 82000 +Processed 1000 documents in this batch, total processed: 83000 +Processed 1000 documents in this batch, total processed: 84000 +Processed 1000 documents in this batch, total processed: 85000 +Processed 1000 documents in this batch, total processed: 86000 +Processed 1000 documents in this batch, total processed: 87000 +Processed 1000 documents in this batch, total processed: 88000 +Processed 1000 documents in this batch, total processed: 89000 +Processed 1000 documents in this batch, total processed: 90000 +Processed 1000 documents in this batch, total processed: 91000 +Processed 1000 documents in this batch, total processed: 92000 + +Error auditing references for document: harvard-g7064-s2-1834-k3 - # +Processed 1000 documents in this batch, total processed: 93000 +Processed 1000 documents in this batch, total processed: 94000 +Processed 1000 documents in this batch, total processed: 95000 +Processed 1000 documents in this batch, total processed: 96000 + +NO MATCH - Duplicate holc-scan.jpg URL +Document: 82a3b7a2-f1b8-4a64-9832-a227ddd1a66d +CSV References Sorted: [["82a3b7a2-f1b8-4a64-9832-a227ddd1a66d", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=staten-island-ny", nil], ["82a3b7a2-f1b8-4a64-9832-a227ddd1a66d", "download", "https://s3.amazonaws.com/holc/tiles/NY/StatenIsland/1938/holc-scan.jpg", "GeoTIFF"], ["82a3b7a2-f1b8-4a64-9832-a227ddd1a66d", "download", "https://s3.amazonaws.com/holc/tiles/NY/StatenIsland/1938/holc-scan.jpg", "JPEG"]] +Document References Sorted: [["82a3b7a2-f1b8-4a64-9832-a227ddd1a66d", "documentation_external", "https://dsl.richmond.edu/panorama/redlining/#city=staten-island-ny", nil], ["82a3b7a2-f1b8-4a64-9832-a227ddd1a66d", "download", "https://s3.amazonaws.com/holc/tiles/NY/StatenIsland/1938/holc-scan.jpg", "JPEG"]] +Processed 1000 documents in this batch, total processed: 97000 +Processed 1000 documents in this batch, total processed: 98000 +Processed 1000 documents in this batch, total processed: 99000 +Processed 1000 documents in this batch, total processed: 100000 +Processed 1000 documents in this batch, total processed: 101000 +Processed 1000 documents in this batch, total processed: 102000 +Processed 1000 documents in this batch, total processed: 103000 + +NO MATCH +Document: test +CSV References Sorted: [["test", "download", "https://example.com", "newLabel"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/kmz_bdry_votingdistricts.zip", "KMZ"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/kmz_bdry_votingdistricts.zip", "newLabel"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/shp_bdry_votingdistricts.zip", "Shapefile"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/shp_bdry_votingdistricts.zip", "anotherLabel"], ["test", "download", "https:/anotherExample.com", "anotherLabel"]] +Document References Sorted: [["test", "download", "https://example.com", "newLabel"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/kmz_bdry_votingdistricts.zip", "KMZ"], ["test", "download", "https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_sos/bdry_votingdistricts/shp_bdry_votingdistricts.zip", "Shapefile"], ["test", "download", "https:/anotherExample.com", "anotherLabel"]] + +NO MATCH - Array of cog URLs? +Document: btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166 +CSV References Sorted: [["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "cog", ["https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/51011e42-c559-47dd-8e18-f6388ecdc083/f2d92d1dfb83d2361547c62c5321be52.tif", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/6b7500e0-acea-45c3-a8b2-47a8b2a5fb20/acc4aa6b5e9ff7f346a6a3f1b13cc2df.tif"], nil], ["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "documentation_external", "http://maps.indiana.edu/layerGallery.html", nil], ["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "download", "http://maps.indiana.edu/download/Environment/Land_Cover_Impervious_Surfaces_2006.zip", "GeoTIFF"]] +Document References Sorted: [["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "cog", "[\"https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/51011e42-c559-47dd-8e18-f6388ecdc083/f2d92d1dfb83d2361547c62c5321be52.tif\", \"https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/6b7500e0-acea-45c3-a8b2-47a8b2a5fb20/acc4aa6b5e9ff7f346a6a3f1b13cc2df.tif\"]", nil], ["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "documentation_external", "http://maps.indiana.edu/layerGallery.html", nil], ["btaa-73cb3ca0-0024-4e7a-b9ba-3902d5c00166", "download", "http://maps.indiana.edu/download/Environment/Land_Cover_Impervious_Surfaces_2006.zip", "GeoTIFF"]] +Processed 1000 documents in this batch, total processed: 104000 +Processed 1000 documents in this batch, total processed: 105000 +Processed 1000 documents in this batch, total processed: 106000 +Processed 1000 documents in this batch, total processed: 107000 +Processed 1000 documents in this batch, total processed: 108000 + +NO MATCH - Array of image URLs? +Document: 59bd8ed8-088c-4932-97c9-544135dd9d64 +CSV References Sorted: [["59bd8ed8-088c-4932-97c9-544135dd9d64", "documentation_external", "https://michiganology.org/uncategorized/IO_59bd8ed8-088c-4932-97c9-544135dd9d64", nil], ["59bd8ed8-088c-4932-97c9-544135dd9d64", "image", ["https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/88b43100-06a1-47e0-8fe0-797b1768f4e0/952fffe1b6f9f9367fdec0cfca0b3c2b.png", "https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/29421b81-8152-4fff-972e-90f98ed57ee2/aa172e2dd422e038b09b0f081e82084e.png"], nil]] +Document References Sorted: [["59bd8ed8-088c-4932-97c9-544135dd9d64", "documentation_external", "https://michiganology.org/uncategorized/IO_59bd8ed8-088c-4932-97c9-544135dd9d64", nil], ["59bd8ed8-088c-4932-97c9-544135dd9d64", "image", "[\"https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/88b43100-06a1-47e0-8fe0-797b1768f4e0/952fffe1b6f9f9367fdec0cfca0b3c2b.png\", \"https://geobtaa-assets-dev.s3.us-east-2.amazonaws.com/store/asset/29421b81-8152-4fff-972e-90f98ed57ee2/aa172e2dd422e038b09b0f081e82084e.png\"]", nil]] +Processed 1000 documents in this batch, total processed: 109000 +Processed 178 documents in this batch, total processed: 109178 +--- Audit End --- diff --git a/docs/references_migration.txt b/docs/references_migration.txt new file mode 100644 index 000000000..27a194a5c --- /dev/null +++ b/docs/references_migration.txt @@ -0,0 +1,21 @@ +# Migration steps + +1. Set `GBL_ADMIN_REFERENCES_MIGRATED=false` in .env.production +2. Deploy code to box +3. Migrate references +4. Audit references +5. Set `GBL_ADMIN_REFERENCES_MIGRATED=true` in .env.production +6. Restart puma and sidekiq via: + +``` +sudo systemctl restart puma +sudo systemctl restart sidekiq +``` + +7. Reindex documents + +``` +RAILS_ENV=production bundle exec rake geoblacklight_admin:solr:reindex +``` + +8. PROFIT! diff --git a/solr/conf/schema.xml b/solr/conf/schema.xml index c8f475539..02343d0ea 100644 --- a/solr/conf/schema.xml +++ b/solr/conf/schema.xml @@ -226,4 +226,7 @@ + + +