Merge pull request #220 from IdentityPython/ft-refactor-resources

refactor resources, adding a lot of typing

Author: leifj

requirements.txt

@@ -8,6 +8,7 @@ lxml >=4.1.1
 mako
 pyXMLSecurity >=0.15
 pyconfig
+pydantic
 pyramid
 pyyaml >=3.10
 redis

src/pyff/api.py

@@ -2,7 +2,7 @@
 import threading
 from datetime import datetime, timedelta
 from json import dumps
-from typing import Any, List, Mapping, Iterator
+from typing import Any, Iterable, List, Mapping
 
 import pkg_resources
 import pyramid.httpexceptions as exc
@@ -389,7 +389,7 @@ def resources_handler(request):
     :return: a JSON representation of the set of resources currently loaded by the server
     """
 
-    def _infos(resources: Iterator[Resource]) -> List[Mapping[str, Any]]:
+    def _infos(resources: Iterable[Resource]) -> List[Mapping[str, Any]]:
         return list(filter(lambda i: 'State' in i and i['State'] is not None, [_info(r) for r in resources]))
 
     def _info(r: Resource) -> Mapping[str, Any]:

src/pyff/builtins.py

@@ -27,6 +27,7 @@
 from .exceptions import MetadataException
 from .logs import get_log
 from .pipes import PipeException, PipelineCallback, Plumbing, pipe
+from .resource import ResourceOpts
 from .samlmd import (
     annotate_entity,
     discojson_t,
@@ -352,7 +353,7 @@ def _pipe(req, *opts):
 
 
 @pipe
-def when(req, condition, *values):
+def when(req: Plumbing.Request, condition: str, *values):
     """
     Conditionally execute part of the pipeline.
 
@@ -377,6 +378,8 @@ def when(req, condition, *values):
     followed. If 'bar' is present in the state with the value 'bill' then the other branch is followed.
     """
     c = req.state.get(condition, None)
+    if c is None:
+        log.debug(f'Condition {repr(condition)} not present in state {req.state}')
     if c is not None and (not values or _any(values, c)):
         return Plumbing(pipeline=req.args, pid="%s.when" % req.plumbing.id).iprocess(req)
     return req.t
@@ -628,32 +631,37 @@ def load(req, *opts):
         )
 
         url = r.pop(0)
-        params = {"via": [], "cleanup": [], "verify": None, "as": url}
+
+        # Copy parent node opts as a starting point
+        child_opts = req.md.rm.opts.copy(update={"via": [], "cleanup": [], "verify": None, "alias": url})
 
         while len(r) > 0:
             elt = r.pop(0)
             if elt in ("as", "verify", "via", "cleanup"):
+                # These elements have an argument
                 if len(r) > 0:
-                    if elt in ("via", "cleanup"):
-                        params[elt].append(r.pop(0))
+                    value = r.pop(0)
+                    if elt == "as":
+                        child_opts.alias = value
+                    elif elt == "verify":
+                        child_opts.verify = value
+                    elif elt == "via":
+                        child_opts.via.append(PipelineCallback(value, req, store=req.md.store))
+                    elif elt == "cleanup":
+                        child_opts.cleanup.append(PipelineCallback(value, req, store=req.md.store))
                     else:
-                        params[elt] = r.pop(0)
+                        raise ValueError(f'Unhandled resource option {elt}')
                 else:
                     raise PipeException(
                         "Usage: load resource [as url] [[verify] verification] [via pipeline]* [cleanup pipeline]*"
                     )
             else:
-                params['verify'] = elt
-
-        if params['via'] is not None:
-            params['via'] = [PipelineCallback(p, req, store=req.md.store) for p in params['via']]
-
-        if params['cleanup'] is not None:
-            params['cleanup'] = [PipelineCallback(p, req, store=req.md.store) for p in params['cleanup']]
+                child_opts.verify = elt
 
-        params.update(opts)
+        # override anything in child_opts with what is in opts
+        child_opts = child_opts.copy(update=opts)
 
-        req.md.rm.add_child(url, **params)
+        req.md.rm.add_child(url, child_opts)
 
     log.debug("Refreshing all resources")
     req.md.rm.reload(fail_on_error=bool(opts['fail_on_error']))

src/pyff/logs.py

@@ -57,7 +57,7 @@ def isEnabledFor(self, lvl):
         return self._log.isEnabledFor(lvl)
 
 
-def get_log(name):
+def get_log(name: str) -> PyFFLogger:
     return PyFFLogger(name)
 
 

src/pyff/parse.py

@@ -1,10 +1,13 @@
 import os
-from datetime import datetime
+from abc import ABC
+from collections import deque
+from typing import Any, AnyStr, Dict, List, Mapping, Optional, Union
 
 from xmlsec.crypto import CertDict
 
 from .constants import NS
 from .logs import get_log
+from .resource import Resource, ResourceOpts
 from .utils import find_matching_files, parse_xml, root, unicode_stream, utc_now
 
 __author__ = 'leifj'
@@ -22,10 +25,22 @@ def raise_wraped(self):
         raise self._wraped
 
 
-class PyffParser(object):
+class PyffParser(ABC):
     def to_json(self):
         return str(self)
 
+    def magic(self, content: str):
+        """Return True if this parser is applicable to this content"""
+        raise NotImplementedError()
+
+    def parse(self, resource: Resource, content: str) -> Mapping[str, Any]:
+        """Initialise/update a resource based on this content, returning information about it
+        TODO: Determine what 'parse' actually means
+
+        TODO: Return something more structured than an arbitrary mapping
+        """
+        raise NotImplementedError()
+
 
 class NoParser(PyffParser):
     def __init__(self):
@@ -34,10 +49,10 @@ def __init__(self):
     def __str__(self):
         return "Not a supported type"
 
-    def magic(self, content):
+    def magic(self, content: str) -> bool:
         return True
 
-    def parse(self, resource, content):
+    def parse(self, resource: Resource, content: str) -> Mapping[str, Any]:
         raise ParserException("No matching parser found for %s" % resource.url)
 
 
@@ -48,17 +63,18 @@ def __init__(self, extensions):
     def __str__(self):
         return "Directory"
 
-    def magic(self, content):
+    def magic(self, content: str) -> bool:
         return os.path.isdir(content)
 
-    def parse(self, resource, content):
-        resource.children = []
+    def parse(self, resource: Resource, content: str) -> Mapping[str, Any]:
+        resource.children = deque()
         info = dict()
         info['Description'] = 'Directory'
         info['Expiration Time'] = 'never expires'
         n = 0
         for fn in find_matching_files(content, self.extensions):
-            resource.add_child("file://" + fn)
+            child_opts = resource.opts.copy(update={'alias': None})
+            resource.add_child("file://" + fn, child_opts)
             n += 1
 
         if n == 0:
@@ -78,10 +94,10 @@ def __init__(self):
     def __str__(self):
         return "XRD"
 
-    def magic(self, content):
+    def magic(self, content: str) -> bool:
         return 'XRD' in content
 
-    def parse(self, resource, content):
+    def parse(self, resource: Resource, content: str) -> Mapping[str, Any]:
         info = dict()
         info['Description'] = "XRD links"
         info['Expiration Time'] = 'never expires'
@@ -97,22 +113,24 @@ def parse(self, resource, content):
                 if len(fingerprints) > 0:
                     fp = fingerprints[0]
                 log.debug("XRD: {} verified by {}".format(link_href, fp))
-                resource.add_child(link_href, verify=fp)
+                child_opts = resource.opts.copy(update={'alias': None})
+                resource.add_child(link_href, child_opts)
         resource.last_seen = utc_now().replace(microsecond=0)
         resource.expire_time = None
         resource.never_expires = True
         return info
 
 
-_parsers = [XRDParser(), DirectoryParser(['xml']), NoParser()]
+_parsers: List[PyffParser] = [XRDParser(), DirectoryParser(['xml']), NoParser()]
 
 
 def add_parser(parser):
     _parsers.insert(0, parser)
 
 
-def parse_resource(resource, content):
+def parse_resource(resource: Resource, content: str) -> Optional[Mapping[str, Any]]:
     for parser in _parsers:
         if parser.magic(content):
             resource.last_parser = parser
             return parser.parse(resource, content)
+    return None