change to using timezone aware utc_now() functionality in Python 3

All times in SAML are supposed to be "without offset" from UTC, and are
typically expressed as ISO8601 formatted strings with a timezone
component of "Z".

Keep doing that, but use the built-in isoformat() and fromisoformat(),
and thoroughly skip microseconds where we might output such times.

This removes the need for the external dependency 'iso8601'.

Author: fredrikt

requirements.txt

@@ -4,7 +4,6 @@ cachetools
 gunicorn
 httplib2 >=0.7.7
 ipaddr
-iso8601 >=0.1.4
 lxml >=4.1.1
 mako
 nose

src/pyff/api.py

@@ -24,7 +24,7 @@
 from .repo import MDRepository
 from .resource import Resource
 from .samlmd import entity_display_name
-from .utils import b2u, dumptree, duration2timedelta, hash_id, json_serializer
+from .utils import b2u, dumptree, duration2timedelta, hash_id, json_serializer, utc_now
 
 log = get_log(__name__)
 
@@ -538,8 +538,7 @@ def mkapp(*args, **kwargs):
         ctx.add_route('request', '/*path', request_method='GET')
         ctx.add_view(request_handler, route_name='request')
 
-        start = datetime.utcnow() + timedelta(seconds=1)
-        log.debug(start)
+        start = utc_now() + timedelta(seconds=1)
         if config.update_frequency > 0:
             ctx.registry.scheduler.add_job(
                 call,

src/pyff/builtins.py

@@ -17,13 +17,12 @@
 import ipaddr
 import six
 import xmlsec
-from iso8601 import iso8601
 from lxml.etree import DocumentInvalid
 from six.moves.urllib_parse import quote_plus, urlparse
 
 from pyff.pipes import registry
 
-from .constants import NS, config
+from .constants import NS
 from .decorators import deprecated
 from .exceptions import MetadataException
 from .logs import get_log
@@ -42,12 +41,15 @@
     sort_entities,
 )
 from .utils import (
+    datetime2iso,
     dumptree,
     duration2timedelta,
     hash_id,
+    iso2datetime,
     root,
     safe_write,
     total_seconds,
+    utc_now,
     validate_document,
     with_tree,
     xslt_transform,
@@ -1503,7 +1505,7 @@ def finalize(req, *opts):
         if name:
             e.set('Name', name)
 
-    now = datetime.utcnow()
+    now = utc_now()
 
     mdid = req.args.get('ID', 'prefix _')
     if re.match('(\s)*prefix(\s)*', mdid):
@@ -1520,17 +1522,16 @@ def finalize(req, *opts):
         offset = duration2timedelta(valid_until)
         if offset is not None:
             dt = now + offset
-            e.set('validUntil', dt.strftime("%Y-%m-%dT%H:%M:%SZ"))
+            e.set('validUntil', datetime2iso(dt))
         elif valid_until is not None:
             try:
-                dt = iso8601.parse_date(valid_until)
-                dt = dt.replace(tzinfo=None)  # make dt "naive" (tz-unaware)
+                dt = iso2datetime(valid_until)
                 offset = dt - now
-                e.set('validUntil', dt.strftime("%Y-%m-%dT%H:%M:%SZ"))
+                e.set('validUntil', datetime2iso(dt))
             except ValueError as ex:
                 log.error("Unable to parse validUntil: %s (%s)" % (valid_until, ex))
 
-                # set a reasonable default: 50% of the validity
+        # set a reasonable default: 50% of the validity
         # we replace this below if we have cacheDuration set
         req.state['cache'] = int(total_seconds(offset) / 50)
 

src/pyff/parse.py

@@ -5,7 +5,7 @@
 
 from .constants import NS
 from .logs import get_log
-from .utils import find_matching_files, first_text, parse_xml, root, unicode_stream
+from .utils import find_matching_files, parse_xml, root, unicode_stream, utc_now
 
 __author__ = 'leifj'
 
@@ -66,7 +66,7 @@ def parse(self, resource, content):
 
         resource.never_expires = True
         resource.expire_time = None
-        resource.last_seen = datetime.now()
+        resource.last_seen = utc_now().replace(microsecond=0)
 
         return dict()
 
@@ -98,7 +98,7 @@ def parse(self, resource, content):
                     fp = fingerprints[0]
                 log.debug("XRD: {} verified by {}".format(link_href, fp))
                 resource.add_child(link_href, verify=fp)
-        resource.last_seen = datetime.now()
+        resource.last_seen = utc_now().replace(microsecond=0)
         resource.expire_time = None
         resource.never_expires = True
         return info

src/pyff/resource.py

@@ -3,12 +3,14 @@
 An abstraction layer for metadata fetchers. Supports both synchronous and asynchronous fetchers with cache.
 
 """
+from __future__ import annotations
 
 import os
 from collections import deque
 from copy import deepcopy
 from datetime import datetime
 from threading import Condition, Lock
+from typing import Optional
 
 import requests
 
@@ -17,7 +19,7 @@
 from .fetch import make_fetcher
 from .logs import get_log
 from .parse import parse_resource
-from .utils import Watchable, hex_digest, img_to_data, non_blocking_lock, url_get
+from .utils import Watchable, hex_digest, img_to_data, non_blocking_lock, url_get, utc_now
 
 requests.packages.urllib3.disable_warnings()
 
@@ -133,9 +135,9 @@ def __init__(self, url=None, **kwargs):
         self.t = None
         self.type = "text/plain"
         self.etag = None
-        self.expire_time = None
-        self.never_expires = False
-        self.last_seen = None
+        self.expire_time: Optional[datetime] = None
+        self.never_expires: bool = False
+        self.last_seen: Optional[datetime] = None
         self.last_parser = None
         self._infos = deque(maxlen=config.info_buffer_size)
         self.children = deque()
@@ -223,7 +225,7 @@ def walk(self):
     def is_expired(self) -> bool:
         if self.never_expires:
             return False
-        now = datetime.now()
+        now = utc_now()
         return self.expire_time is not None and self.expire_time < now
 
     def is_valid(self) -> bool:
@@ -301,7 +303,7 @@ def parse(self, getter):
             info.update(parse_info)
 
         if self.t is not None:
-            self.last_seen = datetime.now()
+            self.last_seen = utc_now().replace(microsecond=0)
             if self.post and isinstance(self.post, list):
                 for cb in self.post:
                     if self.t is not None:

src/pyff/samlmd.py

@@ -17,6 +17,7 @@
     Lambda,
     b2u,
     check_signature,
+    datetime2iso,
     dumptree,
     duration2timedelta,
     filter_lang,
@@ -34,6 +35,7 @@
     subdomains,
     unicode_stream,
     url2host,
+    utc_now,
     validate_document,
     xml_error,
 )
@@ -172,7 +174,10 @@ def parse(self, resource, content):
         )
 
         if expire_time_offset is not None:
-            expire_time = datetime.now() + expire_time_offset
+            now = utc_now()
+            now = now.replace(microsecond=0)
+
+            expire_time = now + expire_time_offset
             resource.expire_time = expire_time
             info['Expiration Time'] = str(expire_time)
 
@@ -217,13 +222,13 @@ def parse(self, resource, content):
             info['NextUpdate'] = next_update
             resource.expire_time = iso2datetime(next_update)
         elif config.respect_cache_duration:
-            now = datetime.utcnow()
+            now = utc_now()
             now = now.replace(microsecond=0)
             next_update = now + duration2timedelta(config.default_cache_duration)
             info['NextUpdate'] = next_update
             resource.expire_time = next_update
 
-        info['Expiration Time'] = str(resource.expire_time)
+        info['Expiration Time'] = 'None' if not resource.expire_time else resource.expire_time.isoformat()
         info['IssuerName'] = first_text(relt, "{%s}IssuerName" % NS['ser'])
         info['SchemeIdentifier'] = first_text(relt, "{%s}SchemeIdentifier" % NS['ser'])
         info['SchemeTerritory'] = first_text(relt, "{%s}SchemeTerritory" % NS['ser'])
@@ -266,7 +271,7 @@ def _update_entities(_t, **kwargs):
                         r.add_via(Lambda(_update_entities, **args))
 
         log.debug("Done parsing eIDAS MetadataServiceList")
-        resource.last_seen = datetime.now()
+        resource.last_seen = utc_now().replace(microsecond=0)
         resource.expire_time = None
         return info
 
@@ -280,10 +285,9 @@ def metadata_expiration(t):
         cache_duration = config.default_cache_duration
         valid_until = relt.get('validUntil', None)
         if valid_until is not None:
-            now = datetime.utcnow()
+            now = utc_now().replace(microsecond=0)
             vu = iso2datetime(valid_until)
-            now = now.replace(microsecond=0)
-            vu = vu.replace(microsecond=0, tzinfo=None)
+            vu = vu.replace(microsecond=0)
             return vu - now
         elif config.respect_cache_duration:
             cache_duration = relt.get('cacheDuration', config.default_cache_duration)
@@ -1037,8 +1041,7 @@ def set_pubinfo(e, publisher=None, creation_instant=None):
         raise MetadataException("At least publisher must be provided")
 
     if creation_instant is None:
-        now = datetime.utcnow()
-        creation_instant = now.strftime("%Y-%m-%dT%H:%M:%SZ")
+        creation_instant = datetime2iso(utc_now())
 
     ext = entity_extensions(e)
     pi = ext.find(".//{%s}PublicationInfo" % NS['mdrpi'])
@@ -1080,10 +1083,8 @@ def expiration(t):
         cache_duration = config.default_cache_duration
         valid_until = relt.get('validUntil', None)
         if valid_until is not None:
-            now = datetime.utcnow()
+            now = utc_now().replace(microsecond=0)
             vu = iso2datetime(valid_until)
-            now = now.replace(microsecond=0)
-            vu = vu.replace(microsecond=0, tzinfo=None)
             return vu - now
         elif config.respect_cache_duration:
             cache_duration = relt.get('cacheDuration', config.default_cache_duration)

src/pyff/utils.py

@@ -26,7 +26,6 @@
 from time import gmtime, strftime
 from typing import Optional, Union
 
-import iso8601
 import pkg_resources
 import requests
 import xmlsec
@@ -177,8 +176,19 @@ def ts_now() -> int:
     return int(time.time())
 
 
-def iso2datetime(s):
-    return iso8601.parse_date(s)
+def iso2datetime(s: str) -> datetime:
+    # TODO: All timestamps in SAML are supposed to be without offset from UTC - raise exception if it is not?
+    if s.endswith('Z'):
+        s = s[:-1] + '+00:00'
+    return datetime.fromisoformat(s)
+
+
+def datetime2iso(dt: datetime) -> str:
+    s = dt.replace(microsecond=0).isoformat()
+    # Use 'Z' instead of +00:00 suffix for UTC times
+    if s.endswith('+00:00'):
+        s = s[:-6] + 'Z'
+    return s
 
 
 def first_text(elt, tag, default=None):
@@ -444,13 +454,16 @@ def valid_until_ts(elt, default_ts: int) -> int:
     ts = default_ts
     valid_until = elt.get("validUntil", None)
     if valid_until is not None:
-        dt = iso8601.parse_date(valid_until)
+        try:
+            dt = datetime.fromtimestamp(valid_until)
+        except Exception:
+            dt = None
         if dt is not None:
             ts = totimestamp(dt)
 
     cache_duration = elt.get("cacheDuration", None)
     if cache_duration is not None:
-        dt = datetime.utcnow() + duration2timedelta(cache_duration)
+        dt = utc_now() + duration2timedelta(cache_duration)
         if dt is not None:
             ts = totimestamp(dt)
 
@@ -951,3 +964,8 @@ def notify(self, *args, **kwargs):
             except BaseException as ex:
                 log.debug(traceback.format_exc())
                 log.warn(ex)
+
+
+def utc_now() -> datetime:
+    """ Return current time with tz=UTC """
+    return datetime.now(tz=timezone.utc)