Merge pull request #14 from cedadev/1.0.1

Updates for 1.0.1

docs/user_guide/source/_templates/layout.html

@@ -0,0 +1,30 @@
+{% extends "!layout.html" %}
+  {% block footer %} {{ super() }}
+
+  <style>
+    /* Sidebar header (and topbar for mobile) */
+    .wy-side-nav-search, .wy-nav-top {
+      background: #A6C36F;
+    }
+
+    .caption-text {
+        color: #828C51
+    }
+
+    .wy-nav-side {
+        background: #1E352F;
+    }
+
+    a {
+        color: #828C51
+    }
+
+    .note {
+        background: #BEEF9E
+    }
+    .admonition-title {
+        background: #335145
+    }
+
+  </style>
+{% endblock %}

docs/user_guide/source/catalog_organisation.rst

@@ -80,7 +80,7 @@ Transactions
 ------------
 
 **Transactions** record the user's action when PUTting a file into the NLDS.
-As alluded to above, in the :ref:`_holding` section, each **holding** can contain
+As alluded to above, in the :ref:`holding` section, each **holding** can contain
 numerous **transactions**.  A **transaction** is created every time a user PUTs
 a single file, or list of files, into the NLDS.  This **transaction** is assigned
 to a holding based on the **label** supplied by the user.  If a **label** is
@@ -150,26 +150,26 @@ The user interacts with the NLDS by PUTting and GETting **files**, without knowi
 (or caring) where those **files** are stored.  From a user view, the **files** are
 stored in the NLDS.  In reality the NLDS first writes the **files** to *object
 storage*.  Later the **files** are backed up to *tape storage*.  When the NLDS
-*object storage* approaches capacity, **files** will be removed from the 
-*object storage* depending on a policy which takes into account several variables,
+*cache* approaches capacity, **files** will be removed from the 
+*cache* depending on a policy which takes into account several variables,
 including when they were last accessed.  If a user subsequently GETs a **file**
-that has removed from the *object storage* then the NLDS will first retrive
-the **file** from the *tape storage* to the *object storage* before copying it
+that has removed from the *cache* then the NLDS will first retrieve
+the **file** from the *tape storage* to the *cache* before copying it
 to the user specified target.
 
 The **location** object in the Catalog database is associated to a file, and
 can have one of three states:
 
-1.  The **file** is held on the *object storage* only.  It will be backed up
-to the *tape storage* later.
+1.  The **file** is held on the *cache* only.  It will be backed up to the 
+*tape storage* later.
 
-2.  The **file** is held on both the *object storage* and *tape storage*.  Users
-can access the file without any staging required by the NLDS.
+2.  The **file** is held on both the *cache* and *tape storage*.  Users can 
+access the file without any staging required by the NLDS.
 
 3.  The **file** is held on the *tape storage* only.  If a user accesses the 
 **file** then the NLDS will *stage* it to the *tape storage*, before completing
 the GET on behalf of the user.  The user does not need to concern themselves
 with the details of this.  However, accessing a file that is stored only on 
-*tape* will take longer than if it was held on *object storage*.
+*tape* will take longer than if it was held on *cache*.
 
 

docs/user_guide/source/conf.py

@@ -3,13 +3,22 @@
 # For the full list of built-in configuration values, see the documentation:
 # https://www.sphinx-doc.org/en/master/usage/configuration.html
 
+
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath('../../../'))
+
+from nlds_client import __version__
+
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
-project = 'Near-Line Data Store'
+project = 'Near-Line Data Store Client'
 copyright = '2024, Centre for Environmental Data Analysis, Science and Technologies Facilities Council, UK Research and Innovation'
-author = 'Neil Massey and Jack Leland'
-version = '1.0.0'
-release = '1.0.0-RC1'
+author = 'Neil Massey & Jack Leland'
+
+version = __version__
+release = __version__
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
@@ -18,6 +27,7 @@
 
 templates_path = ['_templates']
 exclude_patterns = []
+html_favicon = '_images/icon-black.png'
 
 
 
@@ -26,3 +36,9 @@
 
 html_theme = 'sphinx_rtd_theme'
 html_static_path = ['_static']
+
+html_logo = "_images/nlds.png"
+html_theme_options = {
+    'logo_only': False,
+    'display_version': True,
+}

docs/user_guide/source/configuration.rst

@@ -13,7 +13,7 @@ This configuration file is JSON formatted and contains the authentication
 credentials required by:
 
   * The OAuth server
-  * The Object Storage
+  * The Object Storage cache
 
 It also contains the default user and group to use when issuing a request to the
 NLDS.  These can be overriden by the ``-u|--user`` and ``-g|--group`` command

docs/user_guide/source/index.rst

@@ -3,12 +3,12 @@
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
-Near-Line Data Store documentation
-==================================
+Near-Line Data Store - Client documentation
+===========================================
 
 The Near-Line Data Store (NLDS) is a multi-tiered storage solution that uses 
-Object Storage as a front end to a tape library. It catalogs the data as it is
-ingested and permits multiple versions of files. It has a microservice 
+Object Storage as a front end cache to a tape library. It catalogs the data as 
+it is ingested and permits multiple versions of files. It has a microservice 
 architecture using a message broker to communicate between the parts.
 Interaction with NLDS is via a HTTP API, with a Python library and command-line 
 client provided to users for programmatic or interactive use.
@@ -32,10 +32,23 @@ Indices and tables
 * :ref:`modindex`
 * :ref:`search`
 
+
+Acknowledgements
+================
+
 NLDS was developed at the `Centre for Environmental Data Analysis <https://www.ceda.ac.uk>`_
 with support from the ESiWACE2 project. The project ESiWACE2 has received 
 funding from the European Union's Horizon 2020 research and innovation programme
 under grant agreement No 823988. 
 
+.. image:: _images/esiwace2.png
+   :width: 300
+   :alt: ESiWACE2 Project Logo
+
+.. image:: _images/ceda.png
+   :width: 300
+   :alt: CEDA Logo
+
+
 NLDS is Open-Source software with a BSD-2 Clause License.  The license can be
 read :ref:`here <license>`.

docs/user_guide/source/tutorial.rst

@@ -19,21 +19,23 @@ This page is a tutorial on NLDS covering:
 .. _intro:
 
 Introduction to the NLDS
---------------------------------------------------------------
+------------------------
 
 The Near-Line Data Store (NLDS) is a multi-tiered storage solution that uses 
-Object Storage as a front end to a tape library.  Interaction with NLDS is via 
+Object Storage as a front end cache to a tape library.  Interaction with NLDS is via 
 a HTTP API, with a Python library and command-line client provided to users for 
 programmatic or interactive use.  
 
 When a user ingests files to the NLDS, the files are first written to the 
-Object Storage, and a backup is made to tape.  
-When the Object Storage is approaching capacity, a set of policies is 
-interrogated to determine which files will be removed from the Object Storage.  
-When a user GETs a file, the NLDS may have to first transfer the file from 
-tape to the Object Storage, if it has been deleted by the policies.  This in 
-effect creates a multitier of hot (disk), warm (Object Storage) and cold (Tape) 
-storage, with a common interface to all.
+cache, and a backup is made to tape. When the cache is approaching capacity, a 
+set of policies is interrogated to determine which files will be removed from 
+the cache. When a user GETs a file, the NLDS may have to first transfer the file 
+from tape to the cache, if it has been deleted by the policies. This, in effect, 
+creates a multi-tiered system of hot (disk), warm (cache) and cold (Tape) 
+storage, with a common interface to all. Additionally, and more importantly, it 
+takes the decision of storage location out of the user's hands; the user only 
+needs to interact with the service itself rather than manually specifying the 
+movement of data between disk, cache, and tape. 
 
 The NLDS also provides a searchable catalog of files, with the user able to
 assign a label and tags to a collection of files.
@@ -50,9 +52,9 @@ Setting up the NLDS client
 Before using the NLDS, a user must install and set-up the NLDS client.  For the
 NLDS installation on JASMIN, the user must also:
 
-#. Have access to the Object Storage tenancy they wish to write files to
-#. Be a member of the GroupWorkspace they wish to use for writing to tape
-#. Have access to a GroupWorkspace to read and write files to
+#. Have access to the Object Storage tenancy (cache) they wish to write files to
+#. Be a member of, and have read/write access to, the GroupWorkspace they wish 
+   to use for writing to tape
 
 The NLDS client is a command-line program with the name `nlds`.
 To install this, refer to the page :ref:`installation<Installation>`.
@@ -64,11 +66,11 @@ to the page :ref:`configuration<Configuration>`.
 
 On JASMIN, the ``user_name`` and ``user_gws`` fields can be filled in by the
 user.  However, all of the fields in the ``server`` section of the file, and the
-``authentication`` section will need to be provided by the JASMIN system 
-adminstrators.  Fields in the ``object_storage`` section will need to match
-those generated by the Object Storage portal on JASMIN.
-Users can freely change the value of the ``oauth_token_file_location`` field in
-the ``authentication`` section.
+``authentication`` section will need to be provided by the ``init`` command - see the 
+:ref:`relevant seciton<init>` for details. Fields in the ``object_storage`` 
+section will need to match those generated by the `Object Store portal <https://s3-portal.jasmin.ac.uk/login>`_ 
+on JASMIN. Users can freely change the value of the ``oauth_token_file_location`` 
+field in the ``authentication`` section.
 
 .. _first:
 
@@ -81,9 +83,9 @@ and authentication sections of the ``.nlds-config`` file for you - see the
 :ref:`relevant seciton<init>` for details.
 
 Once initialised, the first time ``nlds`` is run on JASMIN the user will be 
-asked for their user name and password. These are the JASMIN user name and 
-password, which the user usually uses to login into JASMIN via the 
-``loginx.jasmin.ac.uk`` servers.
+asked for their username and password. These are the JASMIN username and 
+password, which the user usually uses to login into JASMIN via 
+`one of the login servers <https://help.jasmin.ac.uk/docs/interactive-computing/login-servers/>`_.
 
 The following message will be displayed:
 
@@ -95,7 +97,7 @@ The following message will be displayed:
     Username:
     Password:
 
-After entering your JASMIN user name and password, the OAuth2 token is created
+After entering your JASMIN username and password, the OAuth2 token is created
 at the path pointed to by ``oauth_token_file_location`` in the ``~/.nlds-config``
 file.  The NLDS authorisation system uses refresh tokens to fetch a new token
 when the current one has expired
@@ -208,7 +210,7 @@ To get the return output from the ``put`` command in JSON format, specify the
 ``-j`` option.
 
 When a command is invoked, NLDS will return a summary of the command, including
-the transaction_id.
+the ``transaction_id``.
 
 .. code-block:: text
 
@@ -878,7 +880,7 @@ label for the ``-i`` (holding id) or ``-l`` (existing label) options can be used
                 label   : SmallHolding
                 tags    : {}
 
-.. _tags:
+.. _tutorial_tags:
 
 Adding tags to a file collection (META)
 --------------------------------------------------------------

nlds_client/__init__.py

@@ -0,0 +1 @@
+__version__ = '1.0.1'

nlds_client/clientlib/config.py

@@ -207,7 +207,7 @@ def get_group(config, group):
     "verify_certificates": True,
     "resolve_filenames": True
 }
-def get_option(config, option_name, section_name='option'):
+def get_option(config, option_name, section_name='options'):
     """Get an option from either the config or the DEFAULT_OPTIONS dict."""
     if (section_name in config and
         # Get value from config if option section and option present

nlds_client/clientlib/transactions.py

@@ -190,7 +190,6 @@ def main_loop(url: str,
     # If we're not verifying the certificate we can turn off the warnings about 
     # it
     if not verify:
-        pass
         from urllib3.connectionpool import InsecureRequestWarning
         import warnings
         warnings.filterwarnings("ignore", category=InsecureRequestWarning)
@@ -984,6 +983,7 @@ def init_client(
             method=requests.get,
             allow_redirects=True,
             verify=verify_certificates,
+            authenticate_fl=False,
         )
 
         # If we get to this point then the transaction could not be processed

setup.py

@@ -1,5 +1,6 @@
 import os
 from setuptools import setup, find_packages
+from nlds_client import __version__
 
 with open(os.path.join(os.path.dirname(__file__), 'README.md')) as readme:
     README = readme.read()
@@ -9,7 +10,7 @@
 
 setup(
     name='nlds_client',
-    version='1.0.0',
+    version=__version__,
     packages=find_packages(),
     install_requires=[
         'requests',