Merge pull request #1530 from rzetelskik/docs-myst-parser-1.11

[v1.11] Replace recommonmark with MyST-Parser in docs

.gitignore

@@ -18,3 +18,4 @@ examples/common/grafana/values.yaml
 # additions for docs
 docs/_build
 source/.doctrees
+docs/**/*.pyc

docs/pyproject.toml

@@ -9,14 +9,15 @@ authors = ["ScyllaDB Documentation Contributors"]
 python = "^3.9"
 pyyaml = "6.0.1"
 pygments = "2.15.1"
-recommonmark = "^0.7.1"
 sphinx-scylladb-theme = "~1.6.1"
 sphinx-sitemap = "2.5.1"
 sphinx-autobuild = "2021.3.14"
 Sphinx = "7.2.6"
 sphinx-multiversion-scylla = "~0.3.1"
 sphinx-markdown-tables = "~0.0.17"
 redirects_cli ="~0.1.3"
+myst-parser = "^2.0.0"
+recommonmark = "^0.7.1"
 
 [build-system]
 requires = ["poetry>=0.12"]

docs/source/_ext/scylladb_markdown.py

@@ -0,0 +1,65 @@
+import os
+from recommonmark.transform import AutoStructify
+
+class BaseParser:
+    def __init__(self, app):
+        self.app = app
+
+    def setup(self):
+        raise NotImplementedError
+
+class RecommonmarkParser(BaseParser):
+    def setup(self):
+        try:
+            self.app.setup_extension('recommonmark')
+            self.app.setup_extension('sphinx_markdown_tables')
+
+            self.app.add_config_value('recommonmark_config', {
+                'enable_eval_rst': True,
+                'enable_auto_toc_tree': False,
+            }, True)
+            self.app.add_transform(AutoStructify)
+        except ImportError:
+            raise RuntimeError("recommonmark and sphinx_markdown_tables are not installed!")
+
+class MystParser(BaseParser):
+    def setup(self):
+        try:
+            self.app.setup_extension('myst_parser')
+            self.app.config.myst_enable_extensions = ["colon_fence"]
+
+            # TODO: https://github.com/scylladb/scylla-operator/issues/1421
+            # TODO: https://github.com/scylladb/scylla-operator/issues/1422
+            self.app.config.suppress_warnings = ["myst.xref_missing", "myst.header"]
+
+        except ImportError:
+            raise RuntimeError("myst-parser is not installed")
+
+def select_parser(app, config):
+    if not config.scylladb_markdown_enable:
+        return
+
+    current_version = os.environ.get('SPHINX_MULTIVERSION_NAME', 'stable')
+
+    config.source_suffix = {
+        '.rst': 'restructuredtext',
+        '.md': 'markdown',
+    }
+
+    if current_version in config.scylladb_markdown_recommonmark_versions:
+        parser = RecommonmarkParser(app)
+    else:
+        parser = MystParser(app)
+
+    parser.setup()
+
+def setup(app):
+    app.add_config_value('scylladb_markdown_enable', True, 'env')
+    app.add_config_value('scylladb_markdown_recommonmark_versions', [], 'env')
+    app.connect('config-inited', select_parser)
+
+    return {
+        "version": "0.1",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }

docs/source/conf.py

@@ -1,8 +1,10 @@
 # -*- coding: utf-8 -*-
+import os
+import sys
 from datetime import date
 
-from recommonmark.transform import AutoStructify
 from sphinx_scylladb_theme.utils import multiversion_regex_builder
+sys.path.insert(0, os.path.abspath('./_ext'))
 
 # -- General configuration
 
@@ -16,15 +18,13 @@
     'sphinx.ext.extlinks',
     'sphinx_scylladb_theme',
     'sphinx_multiversion',
-    'recommonmark',
-    'sphinx_markdown_tables',
     "sphinx_sitemap",
+    "scylladb_markdown"
 ]
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
-#
-source_suffix = ['.rst', '.md']
+source_suffix = ['.rst']
 autosectionlabel_prefix_document = True
 
 # The encoding of source files.
@@ -50,15 +50,7 @@
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = True
 
-# Setup Sphinx
-def setup(sphinx):
-    sphinx.add_config_value('recommonmark_config', {
-        'enable_eval_rst': True,
-        'enable_auto_toc_tree': False,
-    }, True)
-    sphinx.add_transform(AutoStructify)
-
-# -- Options for not found extension -------------------------------------------
+# -- Options for not found extension
 
 # Template used to render the 404.html generated by this extension.
 notfound_template =  '404.html'
@@ -71,6 +63,10 @@ def setup(sphinx):
 # Read a YAML dictionary of redirections and generate an HTML file for each
 redirects_file = "./redirections.yaml"
 
+# -- Options for markdown extension
+scylladb_markdown_enable = True
+scylladb_markdown_recommonmark_versions = ['v1.8', 'v1.9', 'v1.10', 'v1.11']
+
 # -- Options for multiversion extension
 # Whitelist pattern for tags (set to None to ignore all tags)
 TAGS = []

docs/source/exposing.md

@@ -3,9 +3,9 @@
 This document explains how ScyllaDB Operator exposes ScyllaClusters in different network setups.
 A ScyllaCluster can be exposed in various network configurations, independently to clients and nodes.
 
-``` note::
+:::{note}
 ScyllaClusters can be only exposed when the ScyllaDB version used version is `>=2023.1` ScyllaDB Enterprise or `>=5.2` ScyllaDB Open Source.
-```
+:::
 
 ## Expose Options
 

docs/source/generic.md

@@ -250,17 +250,17 @@ kubectl create configmap scylla-config -n scylla --from-file=/tmp/scylla.yaml --
 ```
 The operator will then apply the overridable properties `prefer_local` and `dc_suffix` if they are available in the provided mounted file.
 
-``` note::
-  If you want to enable authentication, you first need to adjust ``system_auth`` keyspace replication factor to the number of nodes in the datacenter via cqlsh. It allows you to ensure that the user’s information is kept highly available for the cluster. If ``system_auth`` is not equal to the number of nodes and a node fails, the user whose information is on that node will be denied access.
-  For production environments only use ``NetworkTopologyStrategy``.
+:::{note}
+If you want to enable authentication, you first need to adjust `system_auth` keyspace replication factor to the number of nodes in the datacenter via cqlsh. It allows you to ensure that the user’s information is kept highly available for the cluster. If `system_auth` is not equal to the number of nodes and a node fails, the user whose information is on that node will be denied access.
+For production environments only use `NetworkTopologyStrategy`.
 
-  .. code-block:: console
-
-    kubectl -n scylla exec -it pods/simple-cluster-us-east-1-us-east-1a-0 -c scylla -- cqlsh -e "ALTER KEYSPACE system_auth WITH REPLICATION = {'class' : 'NetworkTopologyStrategy', 'us-east-1' : <replication_factor>};"
-
-  You can read more about enabling authentication in the `Enable authentication <https://opensource.docs.scylladb.com/stable/operating-scylla/security/authentication.html>`_ section of ScyllaDB's documentation.
+```shell
+kubectl -n scylla exec -it pods/simple-cluster-us-east-1-us-east-1a-0 -c scylla -- cqlsh -e "ALTER KEYSPACE system_auth WITH REPLICATION = {'class' : 'NetworkTopologyStrategy', 'us-east-1' : <replication_factor>};"
 ```
 
+You can read more about enabling authentication in the [Enable authentication](https://opensource.docs.scylladb.com/stable/operating-scylla/security/authentication.html) section of ScyllaDB's documentation.
+:::
+
 ## Configure Scylla Manager Agent
 
 The operator creates a second container for each scylla instance that runs [Scylla Manager Agent](https://hub.docker.com/r/scylladb/scylla-manager-agent).
@@ -299,17 +299,16 @@ Having edited and saved the yaml, you can check your cluster's Status and Events
 kubectl -n scylla describe scyllaclusters.scylla.scylladb.com/simple-cluster
 ```
 
-``` note::
-  If you have configured ScyllaDB with ``authenticator`` set to ``PasswordAuthenticator``, you need to manually configure the replication factor of the ``system_auth`` keyspace with every scaling operation.
-
-  .. code-block:: console
+:::{note}
+If you have configured ScyllaDB with `authenticator` set to `PasswordAuthenticator`, you need to manually configure the replication factor of the `system_auth` keyspace with every scaling operation.
 
-    kubectl -n scylla exec -it pods/simple-cluster-us-east-1-us-east-1a-0 -c scylla -- cqlsh -u <username> -p <password> -e "ALTER KEYSPACE system_auth WITH REPLICATION = {'class' : 'NetworkTopologyStrategy', 'us-east-1' : <new_replication_factor>};"
-
-
-  It is recommended to set ``system_auth`` replication factor to the number of nodes in each datacenter.
+```shell
+kubectl -n scylla exec -it pods/simple-cluster-us-east-1-us-east-1a-0 -c scylla -- cqlsh -u <username> -p <password> -e "ALTER KEYSPACE system_auth WITH REPLICATION = {'class' : 'NetworkTopologyStrategy', 'us-east-1' : <new_replication_factor>};"
 ```
 
+It is recommended to set `system_auth` replication factor to the number of nodes in each datacenter.
+:::
+
 ## Benchmark with cassandra-stress
 
 After deploying our cluster along with the monitoring, we can benchmark it using cassandra-stress and see its performance in Grafana. We have a mini cli that generates Kubernetes Jobs that run cassandra-stress against a cluster.

docs/source/gke.md

@@ -23,7 +23,9 @@ cd examples/gke
 # ./gke.sh -u [email protected] -p gke-demo-226716 -z us-west1-b
 ```
 
-:warning: Make sure to pass a ZONE (ex.: us-west1-b) and not a REGION (ex.: us-west1) or it will deploy nodes in each ZONE available in the region.
+:::{warning}
+Make sure to pass a ZONE (ex.: us-west1-b) and not a REGION (ex.: us-west1) or it will deploy nodes in each ZONE available in the region.
+:::
 
 After you deploy, see how you can [benchmark your cluster with cassandra-stress](#benchmark-with-cassandra-stress).
 

docs/source/multidc/eks.md

@@ -41,7 +41,7 @@ nodeGroups:
 ```
 
 Specify the first cluster's configuration file and save it as `cluster-us-east-1.yaml`.
-Refer to [Creating an EKS cluster](../../eks#creating-an-eks-cluster) section of ScyllaDB Operator documentation for the reference of the configuration of node groups.
+Refer to [Creating an EKS cluster](../eks.md#creating-an-eks-cluster) section of ScyllaDB Operator documentation for the reference of the configuration of node groups.
 
 To deploy the first cluster, use the below command:
 ```shell
@@ -66,15 +66,15 @@ Once the cluster is ready, refer to [Deploying Scylla on a Kubernetes Cluster](.
 
 #### Prepare nodes for running ScyllaDB
 
-Then, prepare the nodes for running ScyllaDB workloads and deploy a volume provisioner following the steps described in [Deploying Scylla on EKS](../../eks#prerequisites) in ScyllaDB Operator documentation.
+Then, prepare the nodes for running ScyllaDB workloads and deploy a volume provisioner following the steps described in [Deploying Scylla on EKS](../eks.md#prerequisites) in ScyllaDB Operator documentation.
 
 ### Create the second EKS cluster
 
 Below is the required specification for the second cluster. As was the case with the first cluster, the provided values are only exemplary and can be adjusted according to your needs.
 
-``` caution::
-    It is required that the VPCs of the two EKS clusters have non-overlapping IPv4 network ranges.
-```
+:::{caution}
+It is required that the VPCs of the two EKS clusters have non-overlapping IPv4 network ranges.
+:::
 
 ```yaml
 apiVersion: eksctl.io/v1alpha5

docs/source/multidc/gke.md

@@ -48,9 +48,9 @@ gcloud compute networks subnets create scylladb-us-west1 \
     --secondary-range='cluster=172.17.0.0/16,services=172.18.0.0/20'
 ```
 
-``` caution::
-    It is required that the IPv4 address ranges of the subnets allocated for the GKE clusters do not overlap.
-```
+:::{caution}
+It is required that the IPv4 address ranges of the subnets allocated for the GKE clusters do not overlap.
+:::
 
 Refer to [Create a VPC-native cluster](https://cloud.google.com/kubernetes-engine/docs/how-to/alias-ips) and [Alias IP ranges](https://cloud.google.com/vpc/docs/alias-ip) in GKE documentation for more information about VPC native clusters and alias IP ranges.
 
@@ -79,7 +79,7 @@ gcloud container clusters create scylladb-us-east1 \
     --services-secondary-range-name=services
 ```
 
-Refer to [Creating a GKE cluster](../../gke#creating-a-gke-cluster) section of ScyllaDB Operator documentation for more information regarding the configuration and deployment of additional node pools, including the one dedicated for ScyllaDB nodes.
+Refer to [Creating a GKE cluster](../gke.md#creating-a-gke-cluster) section of ScyllaDB Operator documentation for more information regarding the configuration and deployment of additional node pools, including the one dedicated for ScyllaDB nodes.
 
 You will need to get the cluster's context for future operations. To do so, use the below command:
 ```shell