more typing and smaller fixes

Author: fredrikt

src/pyff/builtins.py

@@ -632,7 +632,7 @@ def load(req, *opts):
         url = r.pop(0)
 
         # Copy parent node opts as a starting point
-        child_opts = req.md.rm.opts.copy(update={"via": [], "cleanup": [], "verify": None, "as": url})
+        child_opts = req.md.rm.opts.copy(update={"via": [], "cleanup": [], "verify": None, "alias": url})
 
         while len(r) > 0:
             elt = r.pop(0)

src/pyff/parse.py

@@ -1,7 +1,7 @@
 import os
 from abc import ABC
 from collections import deque
-from typing import Any, List, Mapping
+from typing import Any, AnyStr, Dict, List, Mapping, Optional, Union
 
 from xmlsec.crypto import CertDict
 
@@ -49,10 +49,10 @@ def __init__(self):
     def __str__(self):
         return "Not a supported type"
 
-    def magic(self, content):
+    def magic(self, content: str) -> bool:
         return True
 
-    def parse(self, resource, content):
+    def parse(self, resource: Resource, content: str) -> Mapping[str, Any]:
         raise ParserException("No matching parser found for %s" % resource.url)
 
 
@@ -66,7 +66,7 @@ def __str__(self):
     def magic(self, content: str) -> bool:
         return os.path.isdir(content)
 
-    def parse(self, resource: Resource, content: str):
+    def parse(self, resource: Resource, content: str) -> Mapping[str, Any]:
         resource.children = deque()
         info = dict()
         info['Description'] = 'Directory'
@@ -128,8 +128,9 @@ def add_parser(parser):
     _parsers.insert(0, parser)
 
 
-def parse_resource(resource: Resource, content: str):
+def parse_resource(resource: Resource, content: str) -> Optional[Mapping[str, Any]]:
     for parser in _parsers:
         if parser.magic(content):
             resource.last_parser = parser
             return parser.parse(resource, content)
+    return None

src/pyff/pipes.py

@@ -7,14 +7,16 @@
 import functools
 import os
 import traceback
-from typing import Any, Callable, Dict, List, Optional
+from typing import Any, Callable, Dict, Iterable, List, Optional
 from typing import Type
 
 import yaml
 from apscheduler.schedulers.background import BackgroundScheduler
+from lxml.etree import ElementTree
 
 from .logs import get_log
 from .repo import MDRepository
+from .store import SAMLStoreBase
 from .utils import PyffException, is_text, resource_string
 
 log = get_log(__name__)
@@ -202,11 +204,12 @@ class Plumbing(object):
     would then be signed (using signer.key) and finally published in /var/metadata/public/metadata.xml
     """
 
-    def __init__(self, pipeline: List[Dict[str, Any]], pid: str):
+    def __init__(self, pipeline: Iterable[Dict[str, Any]], pid: str):
         self._id = pid
         self.pipeline = pipeline
 
-    def to_json(self):
+    def to_json(self) -> Iterable[Dict[str, Any]]:
+        # TODO: to_json seems like the wrong name for this function?
         return self.pipeline
 
     @property
@@ -217,7 +220,7 @@ def id(self) -> str:
     def pid(self) -> str:
         return self._id
 
-    def __iter__(self):
+    def __iter__(self) -> Iterable[Dict[str, Any]]:
         return self.pipeline
 
     def __str__(self):
@@ -245,19 +248,19 @@ def __init__(
                 state = dict()
             if not args:
                 args = []
-            self.plumbing = pl
-            self.md = md
-            self.t = t
+            self.plumbing: Plumbing = pl
+            self.md: MDRepository = md
+            self.t: ElementTree = t
             self._id = None
             self.name = name
-            self.args = args
-            self.state = state
-            self.done = False
-            self._store = store
-            self.scheduler = scheduler
-            self.raise_exceptions = raise_exceptions
+            self.args: Iterable[Dict[str, Any]] = args
+            self.state: Dict[str, Any] = state
+            self.done: bool = False
+            self._store: SAMLStoreBase = store
+            self.scheduler: Optional[BackgroundScheduler] = scheduler
+            self.raise_exceptions: bool = raise_exceptions
             self.exception: Optional[BaseException] = None
-            self.parent = None
+            self.parent: Optional[Plumbing.Request] = None
 
         def scope_of(self, entry_point):
             if 'with {}'.format(entry_point) in self.plumbing.pipeline:
@@ -284,7 +287,7 @@ def set_parent(self, _parent):
             self.parent = _parent
 
         @property
-        def store(self):
+        def store(self) -> SAMLStoreBase:
             if self._store:
                 return self._store
             return self.md.store

src/pyff/resource.py

@@ -10,10 +10,11 @@
 from collections import deque
 from datetime import datetime
 from threading import Condition, Lock
-from typing import Any, Callable, Deque, Dict, List, Optional, TYPE_CHECKING, Tuple
+from typing import Any, Callable, Deque, Dict, List, Optional, TYPE_CHECKING, Tuple, Union
 from urllib.parse import quote as urlescape
 
 import requests
+from lxml.etree import ElementTree
 from requests.adapters import Response
 from pydantic import BaseModel, Field
 
@@ -159,11 +160,11 @@ def to_dict(self) -> Dict[str, Any]:
 class Resource(Watchable):
     def __init__(self, url: Optional[str], opts: ResourceOpts):
         super().__init__()
-        self.url: str = url
-        self.opts = opts
-        self.t = None
-        self.type = "text/plain"
-        self.etag = None
+        self.url: Optional[str] = url
+        self.opts: ResourceOpts = opts
+        self.t: Optional[ElementTree] = None
+        self.type: str = "text/plain"
+        self.etag: Optional[str] = None
         self.expire_time: Optional[datetime] = None
         self.never_expires: bool = False
         self.last_seen: Optional[datetime] = None
@@ -304,12 +305,15 @@ def errors(self):
         else:
             return []
 
-    def load_backup(self):
+    def load_backup(self) -> Optional[str]:
         if config.local_copy_dir is None:
             return None
 
         try:
-            return resource_string(self.local_copy_fn)
+            res = resource_string(self.local_copy_fn)
+            if isinstance(res, bytes):
+                return res.decode('utf-8')
+            return res
         except IOError as ex:
             log.warning(
                 "Caught an exception trying to load local backup for {} via {}: {}".format(
@@ -318,7 +322,7 @@ def load_backup(self):
             )
             return None
 
-    def save_backup(self, data):
+    def save_backup(self, data: Optional[str]) -> None:
         if config.local_copy_dir is not None:
             try:
                 safe_write(self.local_copy_fn, data, True)
@@ -332,6 +336,10 @@ def load_resource(self, getter: Callable[[str], Response]) -> Tuple[Optional[str
 
         log.debug("Loading resource {}".format(self.url))
 
+        if not self.url:
+            log.error(f'No URL for resource {self}')
+            return data, status, info
+
         try:
             r = getter(self.url)
 
@@ -346,7 +354,10 @@ def load_resource(self, getter: Callable[[str], Response]) -> Tuple[Optional[str
 
             if r.ok:
                 data = r.text
-                self.etag = r.headers.get('ETag', None) or hex_digest(r.text, 'sha256')
+                _etag = r.headers.get('ETag', None)
+                if not _etag:
+                    _etag = hex_digest(r.text, 'sha256')
+                self.etag = _etag
             elif self.local_copy_fn is not None:
                 log.warning(
                     "Got status={:d} while getting {}. Attempting fallback to local copy.".format(
@@ -379,12 +390,16 @@ def load_resource(self, getter: Callable[[str], Response]) -> Tuple[Optional[str
 
     def parse(self, getter: Callable[[str], Response]) -> Deque[Resource]:
         data, status, info = self.load_resource(getter)
+
+        if not data:
+            raise ResourceException(f'Nothing to parse when loading resource {self}')
+
         info['State'] = 'Parsing'
         # local import to avoid circular import
         from .parse import parse_resource
 
         parse_info = parse_resource(self, data)
-        if parse_info is not None and isinstance(parse_info, dict):
+        if parse_info is not None:
             info.update(parse_info)
 
         if status != 218:  # write backup unless we just loaded from backup