feat(api): auto uptime monitoring (#595)

* feat(api): adding command to get unmonitored URLs

* feat(api): command to print unmonitored urls, added to infra

* feat(infra): added command to create simple uptime robot alarms

* using subcommands

* feat(infra): added subcommand for API key endpoint monitors

* feat(infra): added command to create monitors for token auth endpoints

* feat(infra): option to allow 404 errors

* remove unused endpoint, update default request typet

* feat(infra): pulling in IGNORE_UNMONITORED_URLS from vars instead of secrets in the workflow and changed IGNORE_UNMONITORED_URLS to be treated as JSON array

---------

Co-authored-by: Gerald Iakobinyi-Pich <[email protected]>

.github/workflows/api-ci-review.yml

@@ -7,7 +7,6 @@ on:
 # on:
 #   workflow_dispatch: # This triggers the workflow manually
 
-
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -62,15 +61,25 @@ jobs:
           SECRET_KEY: secret-test-value
         run: python manage.py check
 
+      - name: URL Monitoring Check
+        working-directory: ./api
+        shell: bash
+        env:
+          CERAMIC_CACHE_SCORER_ID: ""
+          SECRET_KEY: secret-test-value
+          UPTIME_ROBOT_READONLY_API_KEY: ${{ secrets.UPTIME_ROBOT_READONLY_API_KEY }}
+          IGNORE_UNMONITORED_URLS: ${{ vars.IGNORE_UNMONITORED_URLS }}
+        run:
+          python manage.py show_urls -f json > urls.json &&
+          python manage.py get_unmonitored_urls --urls urls.json --base-url https://api.scorer.gitcoin.co --out unmonitored.json --allow-paused True &&
+          [ -f unmonitored.json ] && [ `cat unmonitored.json | wc -m` -eq 2 ]
+
       - name: Run API unittests
         working-directory: ./api
         run: pytest
         env:
           CERAMIC_CACHE_SCORER_ID: ""
           SECRET_KEY: secret-test-value
-          DATABASE_URL: postgres://passport_scorer:passport_scorer_pwd@localhost:5432/passport_scorer
-          DATA_MODEL_DATABASE_URL: postgres://passport_scorer:passport_scorer_pwd@localhost:5432/passport_scorer
-          FF_API_ANALYTICS: on
 
   build-api:
     runs-on: ubuntu-latest

.github/workflows/api-promote-prod.yml

@@ -65,6 +65,19 @@ jobs:
         working-directory: ./api
         run: pip3 install -r requirements.txt
 
+      - name: URL Monitoring Check
+        working-directory: ./api
+        shell: bash
+        env:
+          CERAMIC_CACHE_SCORER_ID: ""
+          SECRET_KEY: secret-test-value
+          UPTIME_ROBOT_READONLY_API_KEY: ${{ secrets.UPTIME_ROBOT_READONLY_API_KEY }}
+          IGNORE_UNMONITORED_URLS: ${{ vars.IGNORE_UNMONITORED_URLS }}
+        run:
+          python manage.py show_urls -f json > urls.json &&
+          python manage.py get_unmonitored_urls --urls urls.json --base-url https://api.scorer.gitcoin.co --out unmanaged.json &&
+          [ -f unmanaged.json ] && [ `cat unmanaged.json | wc -m` -eq 2 ]
+
       - name: Run API unittests
         working-directory: ./api
         run: pytest

.github/workflows/api-promote-staging.yml

@@ -72,6 +72,19 @@ jobs:
           SECRET_KEY: secret-test-value
         run: python manage.py check
 
+      - name: URL Monitoring Check
+        working-directory: ./api
+        shell: bash
+        env:
+          CERAMIC_CACHE_SCORER_ID: ""
+          SECRET_KEY: secret-test-value
+          UPTIME_ROBOT_READONLY_API_KEY: ${{ secrets.UPTIME_ROBOT_READONLY_API_KEY }}
+          IGNORE_UNMONITORED_URLS: ${{ vars.IGNORE_UNMONITORED_URLS }}
+        run:
+          python manage.py show_urls -f json > urls.json &&
+          python manage.py get_unmonitored_urls --urls urls.json --base-url https://api.scorer.gitcoin.co --out unmanaged.json --allow-paused True &&
+          [ -f unmanaged.json ] && [ `cat unmanaged.json | wc -m` -eq 2 ]
+
       - name: Run API unittests
         working-directory: ./api
         run: pytest

api/.env-sample

@@ -49,3 +49,4 @@ CGRANTS_API_TOKEN=abc
 REGISTRY_API_READ_DB=default
 
 STAKING_SUBGRAPH_API_KEY=abc
+UPTIME_ROBOT_READONLY_API_KEY=abc

api/registry/api/v2.py

@@ -4,8 +4,8 @@
 import api_logging as logging
 
 # --- Deduplication Modules
-from account.models import Account, Community
-from django.db.models import Max, Q
+from account.models import Community
+from django.db.models import Q
 from ninja import Router
 from registry.api import common, v1
 from registry.api.schema import (
@@ -15,7 +15,6 @@
     DetailedScoreResponse,
     ErrorMessageResponse,
     SigningMessageResponse,
-    StakeSchema,
     StampDisplayResponse,
     SubmitPassportPayload,
 )
@@ -282,24 +281,6 @@ def get_gtc_stake_legacy(request, address: str, round_id: str):
     return v1.get_gtc_stake_legacy(request, address, round_id)
 
 
-@router.get(
-    "/gtc-stake/{str:address}",
-    # auth=ApiKey(),
-    auth=None,
-    response={
-        200: List[StakeSchema],
-        400: ErrorMessageResponse,
-    },
-    summary="Retrieve GTC stake amounts for the GTC Staking stamp",
-    description="Get self and community GTC stakes for an address",
-)
-def get_gtc_stake(request, address: str) -> List[StakeSchema]:
-    """
-    Get GTC relevant stakes for an address
-    """
-    return v1.get_gtc_stake(request, address)
-
-
 @router.get(
     common.history_endpoint["url"],
     auth=common.history_endpoint["auth"],

api/registry/management/commands/generate_ceramiccache_access_token.py

@@ -0,0 +1,30 @@
+from datetime import timedelta
+from django.core.management.base import BaseCommand
+
+from ceramic_cache.api.v1 import DbCacheToken
+
+
+class LongLivedToken(DbCacheToken):
+    lifetime: timedelta = timedelta(days=100 * 365)
+
+
+class Command(BaseCommand):
+    help = "Removes stamp data and sets score to 0 for users in the provided list"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--address",
+            type=str,
+            help="""Address of the user for whom to generate the access token.""",
+            required=True,
+        )
+
+    def handle(self, *args, **kwargs):
+        address = kwargs["address"].lower()
+        self.stdout.write(f"Generating long-lived access token for {address}")
+
+        token = LongLivedToken()
+        token["did"] = f"did:pkh:eip155:1:{address}"
+
+        self.stdout.write("Access token:")
+        self.stdout.write(f"{token.access_token}")

api/registry/management/commands/get_unmonitored_urls.py

@@ -0,0 +1,188 @@
+from django.core.management.base import BaseCommand, CommandError
+from django.conf import settings
+from http.client import HTTPSConnection
+
+import json
+import re
+
+# Will ignore e.g. anything starting with /admin/
+IGNORED_PATH_ROOTS = [
+    "admin",
+    "social",
+]
+
+# Will ignore exact path matches
+IGNORED_URLS = [
+    "/registry/feature/openapi.json",
+    "/registry/feature/docs",
+    "/registry/feature/scorer/generic",
+]
+
+# This should only be used to get a release out quickly when necessary, after
+# which the URL should be added to the hardcoded IGNORED_URLS above to be
+# ignored in the future
+if settings.IGNORE_UNMONITORED_URLS:
+    IGNORED_URLS.extend(settings.IGNORE_UNMONITORED_URLS)
+
+
+class Command(BaseCommand):
+    help = "Removes stamp data and sets score to 0 for users in the provided list"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--urls",
+            type=str,
+            help="""Local path to a file containing json output of `show_urls` command""",
+            required=True,
+        )
+        parser.add_argument(
+            "--out",
+            type=str,
+            help="Output file, json list of unmonitored paths",
+            required=True,
+        )
+        parser.add_argument(
+            "--base-url",
+            type=str,
+            help="Base URL for the site (uptime robot URLs will be filtered using this)",
+            required=True,
+        )
+        parser.add_argument(
+            "--allow-paused",
+            type=bool,
+            help="Allow paused monitors to be considered monitored (default: False)",
+            default=False,
+        )
+
+    def handle(self, *args, **kwargs):
+        self.stdout.write("Running ...")
+        self.stdout.write(f"args     : {args}")
+        self.stdout.write(f"kwargs   : {kwargs}")
+
+        if not settings.UPTIME_ROBOT_READONLY_API_KEY:
+            raise CommandError("UPTIME_ROBOT_READONLY_API_KEY is not set")
+
+        all_django_urls = self.get_all_urls(kwargs["urls"])
+        django_urls = self.filter_urls(all_django_urls)
+
+        self.stdout.write(f"Total URLs: {len(all_django_urls)}")
+        self.stdout.write(f"Ignoring url path roots: {IGNORED_PATH_ROOTS}")
+        self.stdout.write(f"Ignoring urls: {IGNORED_URLS}")
+        self.stdout.write(f"URLs to check: {len(django_urls)}")
+
+        monitored_urls = self.get_uptime_robot_urls(
+            base_url=kwargs["base_url"], allow_paused=kwargs["allow_paused"]
+        )
+        self.stdout.write(f"Allowing paused monitors: {kwargs['allow_paused']}")
+        self.stdout.write(f"Uptime Robot URLs: {len(monitored_urls)}")
+        self.stdout.write(f"Uptime robot URLs: {json.dumps(monitored_urls, indent=2)}")
+
+        unmonitored_urls = []
+        for django_url in django_urls:
+            url_regex = self.convert_django_url_to_regex(django_url)
+
+            # Weird one-liner but it works. Basically the inner () is a generator
+            # and if anything matches the regex, it will result in a generator that
+            # will yield (return to next()) a value, the index of the matching url.
+            # Otherwise next() will return the default value of None
+            matching_monitored_url_index = next(
+                (i for i, url in enumerate(monitored_urls) if re.match(url_regex, url)),
+                None,
+            )
+
+            if matching_monitored_url_index is None:
+                unmonitored_urls.append(django_url)
+            else:
+                self.stdout.write(
+                    f"Matched: {django_url} to {monitored_urls[matching_monitored_url_index]}"
+                )
+                # remove matching monitored url so it doesn't get matched again
+                monitored_urls.pop(matching_monitored_url_index)
+
+        self.stdout.write(f"Unmonitored URLs: {len(unmonitored_urls)}")
+
+        with open(kwargs["out"], "w") as out_file:
+            json.dump(unmonitored_urls, out_file)
+
+        self.stdout.write("Done")
+
+    def convert_django_url_to_regex(self, url: str):
+        # Have to do sub and then replace because re.sub interprets
+        # the replacement string as an invalid group reference
+
+        # Sub integers
+        url_regex = re.sub("<int:.+?>", "<int>", url).replace("<int>", "\\d+")
+        # Sub strings
+        url_regex = re.sub("<str:.+?>", "<str>", url_regex).replace("<str>", "[^/]+")
+        # Sub untyped params
+        url_regex = re.sub("<[^:]+?>", "<str>", url_regex).replace("<str>", "[^/]+")
+
+        return url_regex + "(\\?|$)"
+
+    def get_uptime_robot_urls(self, base_url: str, allow_paused: bool):
+        limit = 50
+        offset = 0
+
+        monitors = []
+        while True:
+            data = self.uptime_robot_monitors_request(limit, offset)
+
+            total = data["pagination"]["total"]
+            monitors.extend(data["monitors"])
+
+            if len(monitors) >= total:
+                break
+
+            offset += limit
+
+        if base_url.endswith("/"):
+            base_url = base_url[:-1]
+
+        urls = [
+            monitor["url"]
+            for monitor in monitors
+            if (allow_paused or monitor["status"] != 0)
+        ]
+
+        return [url.replace(base_url, "") for url in urls if url.startswith(base_url)]
+
+    def uptime_robot_monitors_request(self, limit, offset):
+        conn = HTTPSConnection("api.uptimerobot.com")
+
+        payload = f"api_key={settings.UPTIME_ROBOT_READONLY_API_KEY}&format=json&limit={limit}&offset={offset}"
+
+        headers = {
+            "content-type": "application/x-www-form-urlencoded",
+            "cache-control": "no-cache",
+        }
+
+        conn.request("POST", "/v2/getMonitors", payload, headers)
+
+        res = conn.getresponse()
+        data = res.read()
+
+        return json.loads(data.decode("utf-8"))
+
+    def get_all_urls(self, urls_file_path):
+        with open(urls_file_path) as urls_file:
+            urls_json = json.load(urls_file)
+
+        return [entry["url"] for entry in urls_json]
+
+    def filter_urls(self, urls):
+        # Remove urls that are just a path with a trailing slash, we
+        # don't use these in our api
+        filtered_urls = [url for url in urls if not re.match(r"^[^?]*/$", url)]
+
+        filtered_urls = [
+            url
+            for url in filtered_urls
+            if not any(url.startswith("/" + root + "/") for root in IGNORED_PATH_ROOTS)
+        ]
+
+        filtered_urls = [
+            url
+            for url in filtered_urls
+            if not any(url == ignored for ignored in IGNORED_URLS)
+        ]
+        return filtered_urls

api/scorer/settings/base.py

@@ -36,6 +36,10 @@
 SECURE_SSL_REDIRECT = env("SECURE_SSL_REDIRECT", default=False)
 SECURE_PROXY_SSL_HEADER = env.json("SECURE_PROXY_SSL_HEADER", default=None)
 
+UPTIME_ROBOT_READONLY_API_KEY = env("UPTIME_ROBOT_READONLY_API_KEY", default="")
+# comma separated list of urls to ignore
+IGNORE_UNMONITORED_URLS = env.json("IGNORE_UNMONITORED_URLS", default=[])
+
 STAKING_SUBGRAPH_API_KEY = env("STAKING_SUBGRAPH_API_KEY", default="api-key")
 
 GENERIC_COMMUNITY_CREATION_LIMIT = env.int(

infra/.example-env

@@ -0,0 +1 @@
+UPTIME_ROBOT_API_KEY=123

infra/README.md

@@ -0,0 +1,3 @@
+# Create monitors in uptime robot
+
+npx tsx scripts/uptime_robot/create_monitor.ts --help