optionally serve up front-end apps for all-in-one scenarios

.gitignore

@@ -25,3 +25,6 @@ localconfig.py
 .whoosh
 
 docs/code
+src/pyff/web
+MANIFEST
+.vscode

npm_modules.txt

@@ -0,0 +1,2 @@
+@sunet/[email protected]
+@theidentityselector/[email protected]

setup.py

@@ -2,14 +2,33 @@
 # -*- encoding: utf-8 -*-
 
 from distutils.core import setup
+from distutils.command.sdist import sdist
+from distutils.dir_util import copy_tree
 from pathlib import PurePath
 from platform import python_implementation
 from typing import List
-
+from tempfile import TemporaryDirectory
 from setuptools import find_packages
 
 __author__ = 'Leif Johansson'
-__version__ = '2.0.0'
+__version__ = '2.1.0dev0'
+
+
+class NPMSdist(sdist):
+    def run(self):
+        import subprocess
+
+        npm_modules = load_requirements(here.with_name('npm_modules.txt'))
+        with TemporaryDirectory() as tmp:
+            for npm_module in npm_modules:
+                subprocess.check_call(['npm', 'install', '--production', '--prefix', tmp, npm_module])
+            for npm_module in npm_modules:
+                (npm_module_path, _, _) = npm_module.rpartition('@')
+                copy_tree(
+                    "{}/node_modules/{}/dist".format(tmp, npm_module_path), './src/pyff/web/{}'.format(npm_module_path)
+                )
+
+        super().run()
 
 
 def load_requirements(path: PurePath) -> List[str]:
@@ -37,6 +56,7 @@ def load_requirements(path: PurePath) -> List[str]:
 
 setup(
     name='pyFF',
+    cmdclass={'sdist': NPMSdist},
     version=__version__,
     description="Federation Feeder",
     long_description=README + '\n\n' + NEWS,
@@ -55,7 +75,7 @@ def load_requirements(path: PurePath) -> List[str]:
     packages=find_packages('src'),
     package_dir={'': 'src'},
     include_package_data=True,
-    package_data={'pyff': ['xslt/*.xsl', 'schema/*.xsd']},
+    package_data={'pyff': ['xslt/*.xsl', 'schema/*.xsd', 'web/**/*']},
     zip_safe=False,
     install_requires=install_requires,
     scripts=['scripts/mirror-mdq.sh'],
@@ -64,6 +84,11 @@ def load_requirements(path: PurePath) -> List[str]:
         'paste.app_factory': ['pyffapp=pyff.wsgi:app_factory'],
         'paste.server_runner': ['pyffs=pyff.wsgi:server_runner'],
     },
-    message_extractors={'src': [('**.py', 'python', None), ('**/templates/**.html', 'mako', None),]},
+    message_extractors={
+        'src': [
+            ('**.py', 'python', None),
+            ('**/templates/**.html', 'mako', None),
+        ]
+    },
     python_requires='>=3.7',
 )

src/pyff/api.py

@@ -2,8 +2,8 @@
 import threading
 from datetime import datetime, timedelta
 from json import dumps
+import os
 from typing import Any, Dict, Generator, Iterable, List, Mapping, Optional, Tuple
-
 import pkg_resources
 import pyramid.httpexceptions as exc
 import pytz
@@ -25,7 +25,7 @@
 from pyff.repo import MDRepository
 from pyff.resource import Resource
 from pyff.samlmd import entity_display_name
-from pyff.utils import b2u, dumptree, hash_id, json_serializer, utc_now
+from pyff.utils import b2u, dumptree, hash_id, json_serializer, utc_now, FrontendApp, resource_filename
 
 log = get_log(__name__)
 
@@ -58,6 +58,12 @@ def robots_handler(request: Request) -> Response:
     )
 
 
+def json_response(data) -> Response:
+    response = Response(dumps(data, default=json_serializer))
+    response.headers['Content-Type'] = 'application/json'
+    return response
+
+
 def status_handler(request: Request) -> Response:
     """
     Implements the /api/status endpoint
@@ -77,9 +83,7 @@ def status_handler(request: Request) -> Response:
         threads=[t.name for t in threading.enumerate()],
         store=dict(size=request.registry.md.store.size()),
     )
-    response = Response(dumps(_status, default=json_serializer))
-    response.headers['Content-Type'] = 'application/json'
-    return response
+    return json_response(_status)
 
 
 class MediaAccept(object):
@@ -392,10 +396,7 @@ def _links(url: str, title: Any = None) -> None:
         for v in request.registry.md.store.attribute(aliases[a]):
             _links('%s/%s' % (a, quote_plus(v)))
 
-    response = Response(dumps(jrd, default=json_serializer))
-    response.headers['Content-Type'] = 'application/json'
-
-    return response
+    return json_response(jrd)
 
 
 def resources_handler(request: Request) -> Response:
@@ -420,10 +421,7 @@ def _info(r: Resource) -> Mapping[str, Any]:
 
         return nfo
 
-    response = Response(dumps(_infos(request.registry.md.rm.children), default=json_serializer))
-    response.headers['Content-Type'] = 'application/json'
-
-    return response
+    return json_response(_infos(request.registry.md.rm.children))
 
 
 def pipeline_handler(request: Request) -> Response:
@@ -433,10 +431,7 @@ def pipeline_handler(request: Request) -> Response:
     :param request: the HTTP request
     :return: a JSON representation of the active pipeline
     """
-    response = Response(dumps(request.registry.plumbings, default=json_serializer))
-    response.headers['Content-Type'] = 'application/json'
-
-    return response
+    return json_response(request.registry.plumbings)
 
 
 def search_handler(request: Request) -> Response:
@@ -496,6 +491,27 @@ def launch_memory_usage_server(port: int = 9002) -> None:
     cherrypy.engine.start()
 
 
+class ExtensionPredicate:
+    def __init__(self, val, info):
+        self.segment_name = val[0]
+        self.extensions = tuple(val[0:])
+
+    def text(self):
+        return "extensions = {}".format(self.extensions)
+
+    phash = text
+
+    def __call__(self, info, request):
+        match = info['match']
+        if match[self.segment_name] == '':
+            return True
+
+        for ext in self.extensions:
+            if match[self.segment_name].endswith(ext):
+                return True
+        return False
+
+
 def mkapp(*args: Any, **kwargs: Any) -> Any:
     md = kwargs.pop('md', None)
     if md is None:
@@ -556,6 +572,23 @@ def mkapp(*args: Any, **kwargs: Any) -> Any:
         ctx.add_route('call', '/api/call/{entry}', request_method=['POST', 'PUT'])
         ctx.add_view(process_handler, route_name='call')
 
+        if config.mdq_browser is not None or config.thiss is not None:
+            ctx.add_route_predicate('ext', ExtensionPredicate)
+
+            if config.mdq_browser is not None and len(config.mdq_browser) == 0:
+                config.mdq_browser = resource_filename('web/@sunet/mdq-browser')
+
+            if config.mdq_browser:
+                log.debug("serving mdq-browser from {}".format(config.mdq_browser))
+                FrontendApp.load('/', 'mdq_browser', config.mdq_browser).add_route(ctx)
+
+            if config.thiss is not None and len(config.thiss) == 0:
+                config.thiss = resource_filename('web/@theidentityselector/thiss')
+
+            if config.thiss:
+                log.debug("serving thiss from {}".format(config.thiss))
+                FrontendApp.load('/thiss/', 'thiss', config.thiss).add_route(ctx)
+
         ctx.add_route('request', '/*path', request_method='GET')
         ctx.add_view(request_handler, route_name='request')
 

src/pyff/constants.py

@@ -9,7 +9,7 @@
 import re
 import sys
 from distutils.util import strtobool
-
+from typing import Tuple, Union, Any
 import pyconfig
 import six
 
@@ -253,15 +253,17 @@ class Config(object):
     version = DummySetting('version', info="Show pyff version information", short='v', typeconv=as_bool)
     module = DummySetting("module", info="load additional plugins from the specified module", short='m')
     alias = DummySetting('alias', info="add an alias to the server - argument must be on the form alias=uri", short='A')
+    env = DummySetting('env', info='add an environment variable to the server', short='E')
 
     # deprecated settings
     google_api_key = S("google_api_key", deprecated=True)
     caching_delay = S("caching_delay", default=300, typeconv=as_int, short='D', deprecated=True)
     proxy = S("proxy", default=False, typeconv=as_bool, deprecated=True)
-    public_url = S("public_url", typeconv=as_string, deprecated=True)
     allow_shutdown = S("allow_shutdown", default=False, typeconv=as_bool, deprecated=True)
     ds_template = S("ds_template", default="ds.html", deprecated=True)
 
+    public_url = S("public_url", typeconv=as_string, info="the public URL of the service - not often needed")
+
     loglevel = S("loglevel", default=logging.WARN, info="set the loglevel")
 
     access_log = S("access_log", cmdline=['pyffd'], info="a log target (file) to use for access logs")
@@ -312,6 +314,15 @@ class Config(object):
         info="a set of aliases to add to the server",
     )
 
+    environ = S(
+        "environ",
+        default=dict(),
+        typeconv=as_dict_of_string,
+        cmdline=['pyffd'],
+        hidden=True,
+        info="a set of environement variables to add to the server",
+    )
+
     base_dir = S("base_dir", info="change to this directory before executing the pipeline")
 
     modules = S("modules", default=[], typeconv=as_list_of_string, hidden=True, info="modules providing plugins")
@@ -460,6 +471,9 @@ class Config(object):
         default="/var/run/pyff/backup",
     )
 
+    mdq_browser = S('mdq_browser', typeconv=as_string, info="the directory where mdq-browser can be found")
+    thiss = S('thiss', typeconv=as_string, info="the directory where thiss-js can be found")
+
     @property
     def base_url(self):
         if self.public_url:
@@ -509,6 +523,14 @@ def help(prg):
 config = Config()
 
 
+def opt_eq_split(s: str) -> Tuple[Any, Any]:
+    for sep in [':', '=']:
+        d = tuple(s.rsplit(sep))
+        if len(d) == 2:
+            return d[0], d[1]
+    return None, None
+
+
 def parse_options(program, docs):
     (short_args, long_args) = config.args(program)
     docs += config.help(program)
@@ -525,6 +547,9 @@ def parse_options(program, docs):
     if config.aliases is None or len(config.aliases) == 0:
         config.aliases = dict(metadata=entities)
 
+    if config.environ is None or len(config.environ) == 0:
+        config.environ = dict()
+
     if config.modules is None:
         config.modules = []
 
@@ -541,6 +566,10 @@ def parse_options(program, docs):
                 assert colon == ':'
                 if a and uri:
                     config.aliases[a] = uri
+            elif o in ('-E', '--env'):
+                (k, v) = opt_eq_split(a)
+                if k and v:
+                    config.environ[k] = v
             elif o in ('-m', '--module'):
                 config.modules.append(a)
             else:

src/pyff/mdq.py

@@ -51,6 +51,7 @@ def main():
         'workers': config.worker_pool_size,
         'loglevel': config.loglevel,
         'preload_app': True,
+        'env': config.environ,
         'daemon': config.daemonize,
         'capture_output': False,
         'timeout': config.worker_timeout,

src/pyff/resource.py

@@ -264,7 +264,7 @@ def local_copy_fn(self):
 
     @property
     def post(
-        self,
+            self,
     ) -> Iterable[Callable]:  # TODO: move classes to make this work -> List[Union['Lambda', 'PipelineCallback']]:
         return self.opts.via
 

src/pyff/utils.py

@@ -6,6 +6,8 @@
 This module contains various utilities.
 
 """
+from __future__ import annotations
+
 import base64
 import cgi
 import contextlib
@@ -26,7 +28,8 @@
 from threading import local
 from time import gmtime, strftime
 from typing import Any, BinaryIO, Callable, Dict, List, Optional, Sequence, Set, Tuple, Union
-
+from pyramid.request import Request as PyramidRequest
+from pyramid.response import Response as PyramidResponse
 import pkg_resources
 import requests
 import xmlsec
@@ -50,6 +53,10 @@
 from pyff.exceptions import *
 from pyff.logs import get_log
 
+from pydantic import BaseModel
+from pyramid.static import static_view
+
+
 etree.set_default_parser(etree.XMLParser(resolve_entities=False))
 
 __author__ = 'leifj'
@@ -980,3 +987,48 @@ def notify(self, *args, **kwargs):
 def utc_now() -> datetime:
     """ Return current time with tz=UTC """
     return datetime.now(tz=timezone.utc)
+
+
+class FrontendApp(BaseModel):
+    url_path: str
+    name: str
+    directory: str
+    dirs: List[str] = []
+    exts: Set[str] = set()
+    env: Dict[str, str] = dict()
+
+    @staticmethod
+    def load(url_path: str, name: str, directory: str, env: Optional[Mapping[str, str]] = None) -> FrontendApp:
+        if env is None:
+            env = config.environ
+        fa = FrontendApp(url_path=url_path, name=name, directory=directory, env=env)
+        with os.scandir(fa.directory) as it:
+            for entry in it:
+                if not entry.name.startswith('.'):
+                    if entry.is_dir():
+                        fa.dirs.append(entry.name)
+                    else:
+                        fn, ext = os.path.splitext(entry.name)
+                        fa.exts.add(ext)
+        return fa
+
+    def env_js_handler(self, request: PyramidRequest) -> PyramidResponse:
+        env_js = "window.env = {" + ",".join([f"{k}: '{v}'" for (k, v) in self.env.items()]) + "};"
+        response = PyramidResponse(env_js)
+        response.headers['Content-Type'] = 'text/javascript'
+        return response
+
+    def add_route(self, ctx):
+        env_route = '{}_env_js'.format(self.name)
+        ctx.add_route(env_route, '/env.js', request_method='GET')
+        ctx.add_view(self.env_js_handler, route_name=env_route)
+        for uri_part in [self.url_path] + [self.url_path + d for d in self.dirs]:
+            route = '{}_{}'.format(self.name, uri_part)
+            path = '{:s}{{sep:/?}}{{path:.*}}'.format(uri_part)
+            ctx.add_route(
+                route,
+                path,
+                request_method='GET',
+                ext=['path'] + list(self.exts),
+            )
+            ctx.add_view(static_view(self.directory, use_subpath=False), route_name=route)