diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 00000000..e84c09a2 --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,48 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Python package + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +jobs: + build: + + strategy: + fail-fast: false + matrix: + os: ["ubuntu-24.04", "ubuntu-22.04", "ubuntu-20.04"] + python: ["3.9", "3.10", "3.11", "3.12"] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Lint with Ruff + run: | + pip install ruff + ruff check --output-format=github . + continue-on-error: true + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f test_requirements.txt ]; then pip install -r test_requirements.txt; fi + pip install coveralls + pip install mypy + python -m pip install --editable . + + + - name: Run tests + run: | + make quick_test + make test_coverage + #make typecheck + diff --git a/NEWS.txt b/NEWS.txt index 0b2f56f6..4ee66a99 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -173,4 +173,9 @@ to sign using HSMs. The only mandatory non-python dependency now is lxml. 2.1.3 ----- +* Release date: ons 10 sep 2024 17:17:10 CET + * Add DiscoveryResponse info to SPs in discojson +* Remove cherrypy imports +* Fix logging +* suport SP trust metadata in an entity attribute as JSON blob diff --git a/requirements.txt b/requirements.txt index 62fe5cd5..5cdc6d04 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -urllib3==1.26.18 +urllib3==1.26.19 pytz accept_types >=0.4.1 apscheduler==3.6.3 @@ -24,3 +24,4 @@ wsgi-intercept xmldiff str2bool setuptools>=70.0.0 # not directly required, pinned by Snyk to avoid a vulnerability +zipp>=3.19.1 # not directly required, pinned by Snyk to avoid a vulnerability \ No newline at end of file diff --git a/src/pyff/api.py b/src/pyff/api.py index 95c57b2f..1050efbf 100644 --- a/src/pyff/api.py +++ b/src/pyff/api.py @@ -487,24 +487,11 @@ def cors_headers(request: Request, response: Response) -> None: event.request.add_response_callback(cors_headers) -def launch_memory_usage_server(port: int = 9002) -> None: - import cherrypy - import dowser - - cherrypy.tree.mount(dowser.Root()) - cherrypy.config.update({'environment': 'embedded', 'server.socket_port': port}) - - cherrypy.engine.start() - - def mkapp(*args: Any, **kwargs: Any) -> Any: md = kwargs.pop('md', None) if md is None: md = MDRepository() - if config.devel_memory_profile: - launch_memory_usage_server() - with Configurator(debug_logger=log) as ctx: ctx.add_subscriber(add_cors_headers_response_callback, NewRequest) diff --git a/src/pyff/builtins.py b/src/pyff/builtins.py index a39501bc..8bdd3a2d 100644 --- a/src/pyff/builtins.py +++ b/src/pyff/builtins.py @@ -31,6 +31,7 @@ from pyff.samlmd import ( annotate_entity, discojson_sp_t, + discojson_sp_attr_t, discojson_t, entitiesdescriptor, find_in_document, @@ -731,7 +732,7 @@ def select(req: Plumbing.Request, *opts): Select a set of EntityDescriptor elements as the working document. :param req: The request - :param opts: Options - used for select alias + :param opts: Options - see Options below :return: returns the result of the operation as a working document Select picks and expands elements (with optional filtering) from the active repository you setup using calls @@ -778,25 +779,60 @@ def select(req: Plumbing.Request, *opts): would terminate the plumbing at select if there are no SPs in the local repository. This is useful in combination with fork for handling multiple cases in your plumbings. - The 'as' keyword allows a select to be stored as an alias in the local repository. For instance + Options are put directly after "select". E.g: .. code-block:: yaml - - select as /foo-2.0: "!//md:EntityDescriptor[md:IDPSSODescriptor]" + - select as /foo-2.0 dedup True: "!//md:EntityDescriptor[md:IDPSSODescriptor]" - would allow you to use /foo-2.0.json to refer to the JSON-version of all IdPs in the current repository. - Note that you should not include an extension in your "as foo-bla-something" since that would make your - alias invisible for anything except the corresponding mime type. + **Options** + Defaults are marked with (*) + - as : The 'as' keyword allows a select to be stored as an alias in the local repository. For instance + + .. code-block:: yaml + + - select as /foo-2.0: "!//md:EntityDescriptor[md:IDPSSODescriptor]" + + would allow you to use /foo-2.0.json to refer to the JSON-version of all IdPs in the current repository. + Note that you should not include an extension in your "as foo-bla-something" since that would make your + alias invisible for anything except the corresponding mime type. + + - dedup : Whether to deduplicate the results by entityID. + + Note: When select is used after a load pipe with more than one source, if dedup is set to True + and there are entity properties that may differ from one source to another, these will be squashed + rather than merged. """ + opt_names = ('as', 'dedup') + if len(opts) % 2 == 0: + _opts = dict(list(zip(opts[::2], opts[1::2]))) + else: + _opts = {} + for i in range(0, len(opts), 2): + if opts[i] in opt_names: + _opts[opts[i]] = opts[i + 1] + else: + _opts['as'] = opts[i] + if i + 1 < len(opts): + more_opts = opts[i + 1:] + _opts.update(dict(list(zip(more_opts[::2], more_opts[1::2])))) + break + + _opts.setdefault('dedup', "True") + _opts.setdefault('name', req.plumbing.id) + _opts['dedup'] = bool(str2bool(_opts['dedup'])) + args = _select_args(req) - name = req.plumbing.id + name = _opts['name'] + dedup = _opts['dedup'] + if len(opts) > 0: if opts[0] != 'as' and len(opts) == 1: name = opts[0] if opts[0] == 'as' and len(opts) == 2: name = opts[1] - entities = resolve_entities(args, lookup_fn=req.md.store.select) + entities = resolve_entities(args, lookup_fn=req.md.store.select, dedup=dedup) if req.state.get('match', None): # TODO - allow this to be passed in via normal arguments @@ -1044,6 +1080,36 @@ def _discojson_sp(req, *opts): return json.dumps(res) +@pipe(name='discojson_sp_attr') +def _discojson_sp_attr(req, *opts): + """ + + Return a json representation of the trust information + + .. code-block:: yaml + discojson_sp_attr: + + SP Entities can carry trust information as a base64 encoded json blob + as an entity attribute with name `https://refeds.org/entity-selection-profile`. + The schema of this json is the same as the one produced above from XML + with the pipe `discojson_sp`, and published at: + + https://github.com/TheIdentitySelector/thiss-mdq/blob/master/trustinfo.schema.json + + :param req: The request + :param opts: Options (unusued) + :return: returns a JSON doc + + """ + + if req.t is None: + raise PipeException("Your pipeline is missing a select statement.") + + res = discojson_sp_attr_t(req) + + return json.dumps(res) + + @pipe def sign(req: Plumbing.Request, *_opts): """ diff --git a/src/pyff/constants.py b/src/pyff/constants.py index 469782ce..c01fa534 100644 --- a/src/pyff/constants.py +++ b/src/pyff/constants.py @@ -264,7 +264,7 @@ class Config(object): allow_shutdown = S("allow_shutdown", default=False, typeconv=as_bool, deprecated=True) ds_template = S("ds_template", default="ds.html", deprecated=True) - loglevel = S("loglevel", default=logging.WARN, info="set the loglevel") + loglevel = S("loglevel", default='WARN', info="set the loglevel") access_log = S("access_log", cmdline=['pyffd'], info="a log target (file) to use for access logs") @@ -290,7 +290,7 @@ class Config(object): caching_enabled = S("caching_enabled", default=True, typeconv=as_bool, info="enable caching?") - no_cashing = N('no_cashing', invert=caching_enabled, short='C', info="disable all caches") + no_caching = N('no_caching', invert=caching_enabled, short='C', info="disable all caches") daemonize = S("daemonize", default=True, cmdline=['pyffd'], info="run in background") @@ -523,7 +523,7 @@ def parse_options(program, docs): sys.exit(2) if config.loglevel is None: - config.loglevel = logging.INFO + config.loglevel = 'INFO' if config.aliases is None or len(config.aliases) == 0: config.aliases = dict(metadata=entities) @@ -536,7 +536,7 @@ def parse_options(program, docs): if o in ('-h', '--help'): print(docs) sys.exit(0) - elif o in ('--version',): + elif o in ('-v', '--version'): print("{} version {}".format(program, pyff_version)) sys.exit(0) elif o in ('-A', '--alias'): diff --git a/src/pyff/fetch.py b/src/pyff/fetch.py index 42830ce3..fcfe3c68 100644 --- a/src/pyff/fetch.py +++ b/src/pyff/fetch.py @@ -96,7 +96,7 @@ def schedule(self, url): :param url: the url to fetch :return: nothing is returned. """ - log.debug("scheduling fetch of {}".format(url)) + log.info("scheduling fetch of {}".format(url)) self.request.put(url) def stop(self): diff --git a/src/pyff/logs.py b/src/pyff/logs.py index 6ab1419e..47cc9443 100644 --- a/src/pyff/logs.py +++ b/src/pyff/logs.py @@ -7,12 +7,6 @@ import six -try: - import cherrypy -except ImportError as e: - logging.debug("cherrypy logging disabled") - cherrypy = None - class PyFFLogger(object): def __init__(self, name=None): @@ -29,9 +23,7 @@ def __init__(self, name=None): } def _l(self, severity, msg): - if cherrypy is not None and '' in cherrypy.tree.apps: - cherrypy.tree.apps[''].log(str(msg), severity=severity) - elif severity in self._loggers: + if severity in self._loggers: self._loggers[severity](str(msg)) else: raise ValueError("unknown severity %s" % severity) diff --git a/src/pyff/samlmd.py b/src/pyff/samlmd.py index 8f1f383d..9be47176 100644 --- a/src/pyff/samlmd.py +++ b/src/pyff/samlmd.py @@ -1,4 +1,6 @@ +import json import traceback +from base64 import b64decode from copy import deepcopy from datetime import datetime, timedelta, timezone from str2bool import str2bool @@ -400,7 +402,7 @@ def filter_or_validate( return t -def resolve_entities(entities, lookup_fn=None): +def resolve_entities(entities, lookup_fn=None, dedup=True): """ :param entities: a set of entities specifiers (lookup is used to find entities from this set) @@ -414,13 +416,21 @@ def _resolve(m, l_fn): else: return l_fn(m) - resolved_entities = dict() # a set won't do since __compare__ doesn't use @entityID + if dedup: + resolved_entities = dict() # a set won't do since __compare__ doesn't use @entityID + else: + resolved_entities = [] for member in entities: for entity in _resolve(member, lookup_fn): entity_id = entity.get('entityID', None) if entity is not None and entity_id is not None: - resolved_entities[entity_id] = entity - return resolved_entities.values() + if dedup: + resolved_entities[entity_id] = entity + else: + resolved_entities.append(entity) + if dedup: + return resolved_entities.values() + return resolved_entities def entitiesdescriptor( @@ -1030,6 +1040,25 @@ def discojson_sp(e, global_trust_info=None, global_md_sources=None): return sp +def discojson_sp_attr(e): + + attribute = "https://refeds.org/entity-selection-profile" + b64_trustinfos = entity_attribute(e, attribute) + if b64_trustinfos is None: + return None + + sp = {} + sp['entityID'] = e.get('entityID', None) + sp['profiles'] = {} + + for b64_trustinfo in b64_trustinfos: + str_trustinfo = b64decode(b64_trustinfo.encode('ascii')) + trustinfo = json.loads(str_trustinfo.decode('utf8')) + sp['profiles'].update(trustinfo['profiles']) + + return sp + + def discojson_sp_t(req): d = [] t = req.t @@ -1041,6 +1070,22 @@ def discojson_sp_t(req): if sp is not None: d.append(sp) + sp = discojson_sp_attr(e) + if sp is not None: + d.append(sp) + + return d + + +def discojson_sp_attr_t(req): + d = [] + t = req.t + + for e in iter_entities(t): + sp = discojson_sp_attr(e) + if sp is not None: + d.append(sp) + return d diff --git a/src/pyff/schema/schema.xsd b/src/pyff/schema/schema.xsd index 5d1f1202..8fc9f5e5 100644 --- a/src/pyff/schema/schema.xsd +++ b/src/pyff/schema/schema.xsd @@ -5,7 +5,7 @@ - + diff --git a/src/pyff/test/data/metadata/test-sp-trustinfo-in-attr.xml b/src/pyff/test/data/metadata/test-sp-trustinfo-in-attr.xml new file mode 100644 index 00000000..b1532fda --- /dev/null +++ b/src/pyff/test/data/metadata/test-sp-trustinfo-in-attr.xml @@ -0,0 +1,114 @@ + + + + + + http://swamid.se/policy/mdrps + + + + + + + + + + + + + + + + + + + + http://www.geant.net/uri/dataprotection-code-of-conduct/v1 + + + ewogICJwcm9maWxlcyI6IHsKICAgICJpbmNvbW1vbi13YXlmaW5kZXIiOiB7CiAgICAgICAic3RyaWN0IjogdHJ1ZSwKICAgICAgICJlbnRpdGllcyI6IFsKICAgICAgICAgewogICAgICAgICAgICJzZWxlY3QiOiAiaHR0cHM6Ly9tZHEuaW5jb21tb24ub3JnL2VudGl0aWVzIiwKICAgICAgICAgICAibWF0Y2giOiAibWRfc291cmNlIiwKICAgICAgICAgICAiaW5jbHVkZSI6IHRydWUKICAgICAgICAgfQogICAgICAgXQogICAgfQogIH0KfQ== + + + + + + + + Carbon Portal authentication service + Kolportalens autentiseringstjänst + Single Sign On for services of ICOS Carbon Portal. Maintained by the Carbon Portal team at Physical Geography department (nateko.lu.se). + Single Sign On tjänst för ICOS Kolportalen. Hanteras av Carbon Portal teamet på INES (nateko.lu.se). + https://cpauth.icos-cp.eu/saml/privacyStatement + https://www.icos-cp.eu/ + https://www.icos-cp.eu/ + https://cpauth.icos-cp.eu/saml/privacyStatement + + + + + cpauth.icos-cp.eu + + CN=cpauth.icos-cp.eu + MIIEJzCCAw+gAwIBAgIJANC3VWNs7fbTMA0GCSqGSIb3DQEBCwUAMIGpMQswCQYD +VQQGEwJTRTERMA8GA1UECAwIU2vDg8KlbmUxDTALBgNVBAcMBEx1bmQxGzAZBgNV +BAoMEklDT1MgQ2FyYm9uIFBvcnRhbDEfMB0GA1UECwwWQXV0aGVudGljYXRpb24g +U2VydmljZTEaMBgGA1UEAwwRY3BhdXRoLmljb3MtY3AuZXUxHjAcBgkqhkiG9w0B +CQEWD2luZm9AaWNvcy1jcC5ldTAeFw0xNTAyMDUxMjI0MzZaFw0yNTAyMDIxMjI0 +MzZaMIGpMQswCQYDVQQGEwJTRTERMA8GA1UECAwIU2vDg8KlbmUxDTALBgNVBAcM +BEx1bmQxGzAZBgNVBAoMEklDT1MgQ2FyYm9uIFBvcnRhbDEfMB0GA1UECwwWQXV0 +aGVudGljYXRpb24gU2VydmljZTEaMBgGA1UEAwwRY3BhdXRoLmljb3MtY3AuZXUx +HjAcBgkqhkiG9w0BCQEWD2luZm9AaWNvcy1jcC5ldTCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAM2QN1jaZJeuPAH+4sVMZKk7vg4JIbUuTMKk0+KIAg5M +XiVsRiEUjY+LtIncrvA/kf2CIySI0WkbwZMjcDd03hNj4kLWhuyxfOCwDO6DsUbG +MbyI6HIYWXJp5ljfEEFgtMqT3dDtD5vwq8h4Zy20ukxOoIokKczrAvn4JjkMsj6Z +0CEAFBC29o4E8PWQbUBgvt6Z+2ao+RHMLD7nZVBx98Occ9KfnYnDDd9Oi1XFe009 +zaSbcqY2RpN8I9hcW/KQf3KnGW5xZ5dr4rhGklCkYr+h0W3xKu+hin8bk91t1Dkr +gaKl/N7M3Oof3k+7ZBlwaV97es5InWCeNgDxCGkBRNsCAwEAAaNQME4wHQYDVR0O +BBYEFDcD7MVudooGaNRYqXBYqQi3VzGxMB8GA1UdIwQYMBaAFDcD7MVudooGaNRY +qXBYqQi3VzGxMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBABS02eZS +weXGMJ2fEIy2JH0VhCbjuX/rz+8Hfh9LjzNb3QwKHuwP83yvPqRulV9FYmvOoK8T +fMou5aW0mZ+QgJNKOrxY5vFxUq6pn3OiYbBu3m1C9ajbU/nx2evzt4+qUwTfHFb+ +ZgXpOtmxRekFzVvGZ18BSPJKwAAqqZ11X7skT/NwEAhbgplVPv9WkDmDzqNvHqQJ +nyRgD2ZqUPU9nEOjGy0gI07dciVcYZQ+CiZeSECIWgQwjDEBDuwMCVAZA6gfdz6C +KJuN+RUSKPEcxPxle1MiB4MU0ei5X4xUbvLWKn9Ok7TOXg2BpnMAv6eON1wVo0Aa +D265cqy6Le/toVg= + + + + + + + + + + + + + + + ICOS Carbon Portal SAML service + ICOS Kolportalens SAML tjänst + + + + + + + ICOS Carbon Portal + ICOS Kolportalen + Carbon Portal + Kolportalen + https://www.icos-cp.eu/ + https://www.icos-cp.eu/ + + + Oleg + Mirzov + mailto:oleg.mirzov@nateko.lu.se + + + Alex + Vermeulen + mailto:alex.vermeulen@nateko.lu.se + + diff --git a/src/pyff/test/data/metadata/test02-sp.xml b/src/pyff/test/data/metadata/test02-sp.xml index 57f3913f..506f8e61 100644 --- a/src/pyff/test/data/metadata/test02-sp.xml +++ b/src/pyff/test/data/metadata/test02-sp.xml @@ -25,6 +25,9 @@ http://www.geant.net/uri/dataprotection-code-of-conduct/v1 https://refeds.org/category/code-of-conduct/v2 + + ewogICJwcm9maWxlcyI6IHsKICAgICJpbmNvbW1vbi13YXlmaW5kZXIiOiB7CiAgICAgICAic3RyaWN0IjogdHJ1ZSwKICAgICAgICJlbnRpdGllcyI6IFsKICAgICAgICAgewogICAgICAgICAgICJzZWxlY3QiOiAiaHR0cHM6Ly9tZHEuaW5jb21tb24ub3JnL2VudGl0aWVzIiwKICAgICAgICAgICAibWF0Y2giOiAibWRfc291cmNlIiwKICAgICAgICAgICAiaW5jbHVkZSI6IHRydWUKICAgICAgICAgfQogICAgICAgXQogICAgfQogIH0KfQ== + diff --git a/src/pyff/test/test_pipeline.py b/src/pyff/test/test_pipeline.py index 3a66f88f..d4c687f4 100644 --- a/src/pyff/test/test_pipeline.py +++ b/src/pyff/test/test_pipeline.py @@ -747,12 +747,50 @@ def test_discojson_sp(self): sp_json = json.load(f) assert 'https://example.com.com/shibboleth' in str(sp_json) + assert len(sp_json) == 2 example_sp_json = sp_json[0] assert 'customer' in example_sp_json['profiles'] customer_tinfo = example_sp_json['profiles']['customer'] assert customer_tinfo['entity'][0] == {'entity_id': 'https://example.org/idp.xml', 'include': True} assert customer_tinfo['entities'][0] == {'select': 'http://www.swamid.se/', 'match': 'registrationAuthority', 'include': True} assert customer_tinfo['fallback_handler'] == {'profile': 'href', 'handler': 'https://www.example.org/about'} + + example_sp_json_2 = sp_json[1] + assert 'incommon-wayfinder' in example_sp_json_2['profiles'] + tinfo = example_sp_json_2['profiles']['incommon-wayfinder'] + assert tinfo['entities'][0] == {'select': 'https://mdq.incommon.org/entities', 'match': 'md_source', 'include': True} + assert tinfo['strict'] + except IOError: + pass + finally: + shutil.rmtree(tmpdir) + + def test_discojson_sp_trustinfo_in_attr(self): + with patch.multiple("sys", exit=self.sys_exit): + tmpdir = tempfile.mkdtemp() + os.rmdir(tmpdir) # lets make sure 'store' can recreate it + try: + self.exec_pipeline(""" +- load: + - file://%s/metadata/test-sp-trustinfo-in-attr.xml +- select +- discojson_sp_attr +- publish: + output: %s/disco_sp_attr.json + raw: true + update_store: false +""" % (self.datadir, tmpdir)) + fn = "%s/disco_sp_attr.json" % tmpdir + assert os.path.exists(fn) + with open(fn, 'r') as f: + sp_json = json.load(f) + + assert 'https://example.com/shibboleth' in str(sp_json) + example_sp_json = sp_json[0] + assert 'incommon-wayfinder' in example_sp_json['profiles'] + tinfo = example_sp_json['profiles']['incommon-wayfinder'] + assert tinfo['entities'][0] == {'select': 'https://mdq.incommon.org/entities', 'match': 'md_source', 'include': True} + assert tinfo['strict'] except IOError: pass finally: