Merge branch 'development' into epos-msl

docker/images/mailpit/build.sh

@@ -4,7 +4,7 @@ set -e
 set -u
 set -x
 
-export MAILPIT_VERSION=1.21.7
+export MAILPIT_VERSION=1.21.8
 export DOCKER_SCAN_SUGGEST=false
 DOCKER_TAG="$1"
 

docs/administration/configuring-yoda.md

@@ -281,7 +281,8 @@ vault_copy_max_retries  | How many times to retry copy to vault on particular fo
 
 Variable                     | Description
 -----------------------------|---------------------
-enable_deposit               | Enable deposit module
+enable_deposit               | Enable deposit module (default: false). If the deposit module is enabled, `enable_data_package_reference` should also be set to True.
+enable_data_package_reference| Enable data package references for vault data packages (default: false). Set to true if deposit module is enabled.
 
 ### Datarequest module configuration
 

docs/development/running-unit-integration-tests.md

@@ -11,15 +11,21 @@ This page explains how to run them locally.
 
 ## Running ruleset unit tests
 
-The ruleset uses the [unittest framework](https://docs.python.org/2.7/library/unittest.html) for the unit tests.
-The test suites can be found in the `unit-tests` subdirectory of the ruleset. Run them using Python 2.7. Example:
+The ruleset uses the [unittest framework](https://docs.python.org/3/library/unittest.html) for the unit tests.
+The test suites can be found in the `unit-tests` subdirectory of the ruleset. Example commands for running them
+manually:
 
 ```bash
+$ python -m venv venv
+$ source venv/bin/activate
+$ python -m pip install --upgrade pip
+$ python -m pip install -r requirements.txt
 $ cd unit-tests
-$ python2 -m unittest unit_tests
-.............................................
+$ export PYTHONPATH=$(cd ../util ; pwd):$PYTHONPATH
+$ python -m unittest unit_tests
+..............................................................
 ----------------------------------------------------------------------
-Ran 45 tests in 0.021s
+Ran 62 tests in 0.009s
 
 OK
 ```

environments/development/allinone/group_vars/allinone.yml

@@ -5,9 +5,6 @@
 ansible_user: vagrant                             # Administrative user on instance for Ansible
 ansible_ssh_private_key_file: vagrant/ssh/vagrant # Path to private key file of administrative user
 
-pgsql_version: 15
-postgresql_perform_db_upgrade: true
-
 # Yoda configuration
 instance: allinone                                 # Name of Yoda instance, as defined in hosts file
 yoda_version: development                          # Git branch, for example: development or release-1.8

environments/development/full/hosts

@@ -13,31 +13,31 @@ full_eus
 
 # Define portal host here:
 [full_portal]
-portal
+portal.yoda.test
 
 # Define davrods host here:
 [full_davrods]
-portal
+portal.yoda.test
 
 # Define database host here:
 [full_database]
-database
+database.yoda.test
 
 # Define icat host here:
 [full_icat]
-icat
+icat.yoda.test
 
 # Define resource host here:
 [full_resource]
-resource
+resource.yoda.test
 
 # Define public host here:
 [full_public]
-public
+public.yoda.test
 
 # Define external user service host here:
 [full_eus]
-eus
+eus.yoda.test
 
 ###############
 # All instances

roles/irods_gocommands/defaults/main.yml

@@ -4,5 +4,5 @@
 irods_service_account: irods
 
 irods_enable_gocommands: false
-irods_gocommands_version: 0.10.7
-irods_gocommands_archive_checksum: "sha256:1569dfb4784a1f8ba6ce8d035cc16b439ee2c95f258a30880c1e546a8d58fb0b"
+irods_gocommands_version: 0.10.8
+irods_gocommands_archive_checksum: "sha256:43f80da3f1901239eed7c360b328e57a54b9c287f1a7d960a10a988078827542"

roles/irods_icat/tasks/main.yml

@@ -45,7 +45,6 @@
     name:
       - "{{ irods_idp_package_new }}"
       - "{{ irods_esp_package_new }}"
-#      - "{{ irods_dtp_package_new }}"
     state: present
   when: not ansible_check_mode and enable_open_search
 

roles/irods_icat/templates/oidc.py.j2

@@ -8,10 +8,6 @@ USERINFO_URI = "{{ oidc_userinfo_uri | default('') }}"
 EMAIL_FIELD = "{{ oidc_email_field | default('email') }}"
 
 
-def unescape_irods_pam_password(password):
-    return password.replace('\@', '@')
-
-
 def validate_token(username, token, sub):
     # Send token validation request.
     headers = {"Authorization": "Bearer {}".format(token)}
@@ -58,8 +54,6 @@ def pam_sm_authenticate(pamh, flags, argv):
         print('Missing token for user {}'.format(username))
         return pamh.PAM_AUTH_ERR
 
-    token = unescape_irods_pam_password(token)
-
     # Remove the prefix from the portal.
     if token[0:14] == '++oidc_token++':
         token = token[14:]

roles/irods_icat/templates/setup_irods_provider.json.j2

@@ -29,14 +29,16 @@
   },
   "server_config": {
     "advanced_settings": {
-      "default_log_rotation_in_days": 5,
       "default_number_of_transfer_threads": 4,
       "default_temporary_password_lifetime_in_seconds": 120,
-      "maximum_number_of_concurrent_rule_engine_server_processes": 4,
+      "delay_rule_executors": [],
+      "delay_server_sleep_time_in_seconds" : 30,
       "maximum_size_for_single_buffer_in_megabytes": 32,
+      "maximum_size_of_delay_queue_in_bytes": 0,
       "maximum_temporary_password_lifetime_in_seconds": 1000,
-      "rule_engine_server_execution_time_in_seconds": 120,
-      "rule_engine_server_sleep_time_in_seconds": 10,
+      "migrate_delay_server_sleep_time_in_seconds": 5,
+      "number_of_concurrent_delay_rule_executors": 4,
+      "stacktrace_file_processor_sleep_time_in_seconds": 10,
       "transfer_buffer_size_for_parallel_transfer_in_megabytes": 4,
       "transfer_chunk_size_for_parallel_transfer_in_megabytes": 40
     },
@@ -47,7 +49,6 @@
     "default_dir_mode": "0750",
     "default_file_mode": "0600",
     "default_hash_scheme": "SHA256",
-    "default_resource_name": "{{ irods_default_resc }}",
     "environment_variables": {},
     "federation": [],
     "host_resolution": {

roles/irods_resource/tasks/main.yml

@@ -87,7 +87,7 @@
 
 - name: Configure iRODS resource server
   become: true
-  ansible.builtin.command: python /var/lib/irods/scripts/setup_irods.py --json_configuration_file=/etc/irods/setup_irods_consumer.json
+  ansible.builtin.command: python3 /var/lib/irods/scripts/setup_irods.py --json_configuration_file=/etc/irods/setup_irods_consumer.json
   args:
     creates: /etc/irods/service_account.config
   when: not ansible_check_mode

roles/irods_resource/templates/core.re.j2

@@ -1,15 +1,10 @@
 # {{ ansible_managed }}
-# iRODS Rule Base
-#  The new rule language is used to express all policies
-#  Recovery procedures are included for a micro-service after " ::: "
+# iRODS core.re Rule Base
 #
-#Test Rules
-printHello { print_hello; }
-
-#
-#
-# These are sys admin rules for creating and deleting users and renaming
-# the local zone.
+# Defines the SSL (client-server negotiation) policy for this server
+# - CS_NEG_REFUSE - No SSL
+# - CS_NEG_DONT_CARE - Depends on the Client request
+# - CS_NEG_REQUIRE - Require SSL
 acPreConnect(*OUT) { *OUT="CS_NEG_DONT_CARE"; }
 
 acCreateUser {
@@ -55,8 +50,8 @@ acDeleteCollByAdminIfPresent(*parColl,*childColl) {
 acDeleteCollByAdmin(*parColl,*childColl) {
   msiDeleteCollByAdmin(*parColl,*childColl); }
 #
-acRenameLocalZone(*oldZone,*newZone) {
-  msiRenameCollection("/"++str(*oldZone)++"",*newZone) ::: msiRollback;
+acRenameLocalZone(*oldZone, *newZone) {
+  msiRenameLocalZoneCollection(*newZone) ::: msiRollback;
   msiRenameLocalZone(*oldZone,*newZone) ::: msiRollback;
   msiCommit; }
 #
@@ -163,8 +158,8 @@ acCheckPasswordStrength(*password) { }
 # acSetRescSchemeForCreate {msiSetDefaultResc("demoResc","null"); msiSetRescSortScheme("random"); msiSetRescSortScheme("byRescClass"); }
 # acSetRescSchemeForCreate {msiSetDefaultResc("demoResc7%demoResc8","preferred"); }
 # acSetRescSchemeForCreate {ON($objPath like "/tempZone/home/rods/protected/*") {msiOprDisallowed;} }
-acSetRescSchemeForCreate {msiSetDefaultResc("{{ irods_default_resc }}","null"); }
-acSetRescSchemeForRepl {msiSetDefaultResc("{{ irods_default_resc }}","null"); }
+acSetRescSchemeForCreate {msiSetDefaultResc("{{ irods_default_resc }}","forced"); }
+acSetRescSchemeForRepl {msiSetDefaultResc("{{ irods_default_resc }}","forced"); }
 # acSetRescSchemeForCreate {msiGetSessionVarValue("all","all"); msiSetDefaultResc("demoResc","null"); }
 # acSetRescSchemeForCreate {msiSetDefaultResc("demoResc","forced"); msiSetRescSortScheme("random"); msiSetRescSortScheme("byRescClass"); }
 #
@@ -180,22 +175,10 @@ acSetRescSchemeForRepl {msiSetDefaultResc("{{ irods_default_resc }}","null"); }
 #    msiSetDataObjAvoidResc(avoidResc) - set the resource to avoid when
 #      opening an object. The copy stored in this resource will not be picked
 #      unless this is the only copy.
-#    msiSortDataObj(sortingScheme) - Sort the copies of the data object using
-#      this scheme. Currently, "random" and "byRescClass" sorting scheme are
-#      supported. If "byRescClass" is set, data objects in the "cache"
-#      resources will be placed ahead of of those in the "archive" resources.
-#      The sorting schemes can also be chained. e.g.,
-#      msiSortDataObj(random); msiSortDataObj(byRescClass) means that
-#      the data objects will be sorted randomly first and then separated
-#      by class.
-#    msiStageDataObj(cacheResc) - stage a copy of the data object in the
-#      cacheResc before opening the data object.
 #    The $writeFlag session variable has been created to be used as a condition
 #    for differentiating between open for read ($writeFlag == "0") and
 #    write ($writeFlag == "1"). e.g. :
-# acPreprocForDataObjOpen {ON($writeFlag == "0") {msiStageDataObj("demoResc8"); } }
 # acPreprocForDataObjOpen {ON($writeFlag == "1") { } }
-# acPreprocForDataObjOpen {msiSortDataObj("random"); msiSetDataObjPreferredResc("xyz%demoResc8%abc"); msiStageDataObj("demoResc8"); }
 # acPreprocForDataObjOpen {msiSetDataObjPreferredResc("demoResc7%demoResc8"); }
 acPreprocForDataObjOpen { }
 # acPreprocForDataObjOpen {msiGetSessionVarValue("all","all"); }
@@ -222,30 +205,12 @@ acSetMultiReplPerResc { }
 # in sequence by these rules.
 #    msiExtractNaraMetadata - extract and register metadata from the just
 #     upload NARA files.
-#    msiSysReplDataObj(replResc, flag) - can be used to replicate a copy of
-#    the file just uploaded or copied data object to the specified replResc
-#    Valid values for the "flag" input are "all", "updateRepl" and
-#    "rbudpTransfer". More than one flag values can be set using the
-#    "%" character as separator. e.g., "all%updateRepl". "updateRepl" means
-#    update an existing stale copy to the latest copy. The "all" flag means
-#    update all stale
-#    copies if the "updateRepl" flag is also set. "rbudpTransfer" means
-#    the RBUDP protocol will be used for the transfer.
-#
-#    msiSysChksumDataObj - checksum the just uploaded or copied data object.
-# acPostProcForPut {msiSysChksumDataObj; msiSysReplDataObj("demoResc8","all"); }
-# acPostProcForPut {msiSysReplDataObj("demoResc8","all"); }
-# acPostProcForPut {msiSysChksumDataObj; }
-# acPostProcForPut {delay("<A></A>") {msiSysReplDataObj('demoResc8','all'); } }
 # acWriteLine(*A,*B) {writeLine(*A,*B); }
 # acPostProcForPut {delay("<PLUSET>1m</PLUSET>") {acWriteLine('serverLog','delayed by a minute message1'); acWriteLine('serverLog','delayed by a minute message2'); } }
-# acPostProcForPut {ON($objPath like "/tempZone/home/rods/nvo/*") {delay("<PLUSET>1m</PLUSET>") {msiSysReplDataObj('nvoReplResc','null'); } } }
-# acPostProcForPut {msiSysReplDataObj("demoResc8","all"); }
 #acPostProcForPut {msiSetDataTypeFromExt; }
-#acPostProcForPut {ON($objPath like "/tempZone/home/rods/tg/*") {msiSysReplDataObj("nvoReplResc","null"); } }
 #acPostProcForPut {ON($objPath like "/tempZone/home/rods/mytest/*") {writeLine("serverLog","File Path is "++$filePath); } }
 #acPostProcForPut {ON($objPath like "/tempZone/home/rods/mytest/*") {writeLine("serverLog","File Path is "++$filePath); msiSplitPath($filePath,*fileDir,*fileName); msiExecCmd("send.sh", "*fileDir *fileName", "null", "null","null",*Junk); writeLine("serverLog","After File Path is *fileDir *fileName"); } }
-#acPostProcForPut { ON($objPath like "\*txt") {writeLine("serverLog","File $objPath"); } }
+# acPostProcForPut { ON($objPath like "\*txt") {writeLine("serverLog","File $objPath"); } }
 acPostProcForPut { }
 acPostProcForCopy { }
 acPostProcForFilePathReg { }
@@ -374,15 +339,6 @@ acChkHostAccessControl { }
 # acSetVaultPathPolicy {msiSetRandomScheme; }
 acSetVaultPathPolicy {msiSetGraftPathScheme("no","1"); }
 #
-# 17) acSetReServerNumProc - This rule set the policy for the number of processes
-# to use when running jobs in the irodsReServer. The irodsReServer can now
-# multi-task such that one or two long running jobs cannot block the execution
-# of other jobs. One function can be called:
-#    msiSetReServerNumProc(numProc) - numProc can be "default" or a number
-#    in the range 1-4. numProc will be set to 1 if "default" is the input.
-#
-acSetReServerNumProc {msiSetReServerNumProc("default"); }
-#
 # 18) acPreProcForCollCreate - This is the PreProcessing rule for creating
 # a collection. Currently there is no function written specifically
 # for this rule.
@@ -699,61 +655,30 @@ acPreProcForExecCmd(*cmd, *args, *addr, *hint) { }
 acPreProcForServerPortal(*oprType, *lAddr, *lPort, *pAddr, *pPort, *load) { }
 acPostProcForServerPortal(*oprType, *lAddr, *lPort, *pAddr, *pPort, *load) { }
 acPreProcForWriteSessionVariable(*var) {
-    on(*var == "status") {
-        succeed;
-    }
-    or {
-        failmsg(-1, "Update session variable $*var not allowed!");
-    }
+	on(*var == "status") {
+		succeed;
+	}
+	or {
+		failmsg(-1, "Update session variable $*var not allowed!");
+	}
 }
 
 getSessionVar(*name, *output) {
     *output = eval("str($"++*name++")");
 }
 
-# ----------------------------------------------------------------------------
-# These rules are for testing only
-#acDataObjCreate {acSetCreateConditions; acDOC; }
-acSetCreateConditions {msiGetNewObjDescriptor ::: recover_msiGetNewObjDescriptor; acSetResourceList; }
-acDOC {msiPhyDataObjCreate ::: recover_msiPhyDataObjCreate; acRegisterData ::: msiRollback; msiCommit; }
-acSetResourceList {msiSetResourceList; }
-acSetCopyNumber {msiSetCopyNumber; }
-acRegisterData {msiRegisterData ::: msiRollback; }
-#
-#These are actions for getting iCAT results for performing iRODS operations.
-#These rules generate the genQueryOut_ structure for each action for the given condition
-#
-acGetIcatResults(*Action,*Condition,*GenQOut) {ON((*Action == "replicate") %% (*Action == "trim") %% (*Action == "chksum") %% (*Action == "copy") %% (*Action == "remove")) {msiMakeQuery("DATA_NAME, COLL_NAME",*Condition,*Query); msiExecStrCondQuery(*Query, *GenQOut); cut; } }
-acGetIcatResults(*Action,*Condition,*GenQOut) {ON(*Action == "chksumRescLoc") {msiMakeQuery("DATA_NAME, COLL_NAME, RESC_LOC",*Condition,*Query); msiExecStrCondQuery(*Query, *GenQOut); cut; } }
-acGetIcatResults(*Action,*Condition,*GenQOut) {ON(*Action == "list") {msiMakeQuery("DATA_NAME, COLL_NAME, DATA_RESC_NAME, DATA_REPL_NUM, DATA_SIZE",*Condition,*Query); msiExecStrCondQuery(*Query, *GenQOut); cut; } }
-#
-#rules for purging a file which have expired
-#
-acPurgeFiles(*Condition) {ON((*Condition == "null") %% (*Condition == "")) {msiGetIcatTime(*Time,"unix"); acGetIcatResults("remove","DATA_EXPIRY < '*Time'",*List); foreach(*List) {msiDataObjUnlink(*List,*Status); msiGetValByKey(*List,"DATA_NAME",*D); msiGetValByKey(*List,"COLL_NAME",*E); writeLine("stdout","Purged File *E/*D at *Time"); } } }
-acPurgeFiles(*Condition) {msiGetIcatTime(*Time,"unix"); acGetIcatResults("remove","DATA_EXPIRY < '*Time' AND *Condition",*List); foreach(*List) {msiDataObjUnlink(*List,*Status); msiGetValByKey(*List,"DATA_NAME",*D); msiGetValByKey(*List,"COLL_NAME",*E); writeLine("stdout","Purged File *E/*D at *Time"); } }
-acConvertToInt(*R) {assign(*A,$sysUidClient); assign($sysUidClient,*R); assign(*K, $sysUidClient); assign(*R,*K); assign($sysUidClient,*A); }
-
-#
-#  rule for running a workflow
-#
-acRunWorkFlow(*File, *R_BUF) {
-   msiDataObjOpen("objPath=*File++++openFlags=O_RDONLY",*S_FD);
-   msiDataObjRead(*S_FD,33554412,*R_BUF);
-   msiDataObjClose(*S_FD,*Status2);
-}
-
 acPostProcForParallelTransferReceived(*leaf_resource) {}
 
 acPostProcForDataCopyReceived(*leaf_resource) {}
 
 # =-=-=-=-=-=-=-
 # examples of API dynamic policy enforcement points
-# pep_api_data_obj_put_pre(*COMM, *DATAOBJINP, *BUFFER, *PORTAL_OPR_OUT) {  }
-# pep_api_data_obj_put_post(*COMM, *DATAOBJINP, *BUFFER, *PORTAL_OPR_OUT) {  }
+# pep_api_data_obj_put_pre(*INSTANCE_NAME, *COMM, *DATAOBJINP, *BUFFER, *PORTAL_OPR_OUT) {  }
+# pep_api_data_obj_put_post(*INSTANCE_NAME, *COMM, *DATAOBJINP, *BUFFER, *PORTAL_OPR_OUT) {  }
 
 # =-=-=-=-=-=-=-
 # policy controlling when a dataObject is staged to cache from archive in a compound coordinating resource
-#  - the default is to stage when cache is not present ("when_necessary")
+#  - the default is to stage when cache is stale or not present ("when_necessary")
 # =-=-=-=-=-=-=-
 # pep_resource_resolve_hierarchy_pre(*INSTANCE, *CONTEXT, *OUT, *OPERATION, *HOST, *PARSER, *VOTE){*OUT="compound_resource_cache_refresh_policy=when_necessary";}  # default
 # pep_resource_resolve_hierarchy_pre(*INSTANCE, *CONTEXT, *OUT, *OPERATION, *HOST, *PARSER, *VOTE){*OUT="compound_resource_cache_refresh_policy=always";}

roles/irods_resource/templates/setup_irods_consumer.json.j2

@@ -29,14 +29,16 @@
   },
   "server_config": {
     "advanced_settings": {
-      "default_log_rotation_in_days": 5,
       "default_number_of_transfer_threads": 4,
       "default_temporary_password_lifetime_in_seconds": 120,
-      "maximum_number_of_concurrent_rule_engine_server_processes": 4,
+      "delay_rule_executors": [],
+      "delay_server_sleep_time_in_seconds" : 30,
       "maximum_size_for_single_buffer_in_megabytes": 32,
+      "maximum_size_of_delay_queue_in_bytes": 0,
       "maximum_temporary_password_lifetime_in_seconds": 1000,
-      "rule_engine_server_execution_time_in_seconds": 120,
-      "rule_engine_server_sleep_time_in_seconds": 10,
+      "migrate_delay_server_sleep_time_in_seconds": 5,
+      "number_of_concurrent_delay_rule_executors": 4,
+      "stacktrace_file_processor_sleep_time_in_seconds": 10,
       "transfer_buffer_size_for_parallel_transfer_in_megabytes": 4,
       "transfer_chunk_size_for_parallel_transfer_in_megabytes": 40
     },
@@ -47,9 +49,11 @@
     "default_dir_mode": "0750",
     "default_file_mode": "0600",
     "default_hash_scheme": "SHA256",
-    "default_resource_name": "{{ irods_default_resc }}",
     "environment_variables": {},
     "federation": [],
+    "host_resolution": {
+      "host_entries": []
+    },
     "match_hash_policy": "compatible",
     "negotiation_key": "{{ irods_negotiation_key }}",
     "plugin_configuration": {
@@ -103,5 +107,7 @@
     "zone_name": "{{ irods_zone }}",
     "zone_port": {{ irods_icat_port }},
     "zone_user": "rods"
-  }
+  },
+  "default_resource_name": "{{ irods_default_resc }}",
+  "default_resource_directory": ""
 }