Resolve redirects when testing URLs for equality

The URL DOIs resolve to can move around, with redirects pointing to the
new location. To make the tests more robust, only fail if the URLs
differ after redirections.

See also
https://www.crossref.org/blog/urls-and-dois-a-complicated-relationship/

Author: hseg

tests/test_doi.py

@@ -3,6 +3,10 @@
 import os
 from pkg_resources import parse_version
 
+from urllib.request import Request, urlopen
+from urllib.parse import urlparse, urlunparse
+from warnings import warn
+
 import pytest
 
 from doi import (
@@ -11,6 +15,30 @@
 )
 
 
+def simplify_url(u):
+    return urlparse(u)._replace(query='', fragment='')
+
+
+def resolve_redirects(u):
+    # Unconditionally upgrade to https, since some resolvers seem to require it
+    # If removed, it'd make sense to canonicalize in simplify_url instead to
+    # prevent spurious test failures
+    u = urlunparse(urlparse(u)._replace(scheme='https'))
+    req = Request(u, headers={'User-Agent': 'Mozilla/5.0'})
+    with urlopen(req) as r:
+        return simplify_url(r.url)
+
+
+def normalize_eq(u, v):
+    if u == v:
+        return True
+    warn(f"{u} textually differs from {v}, please update the relevant case.\n"
+        "Attempting to recover by resolving redirects")
+    return (simplify_url(u) == simplify_url(v)
+            or resolve_redirects(u) == resolve_redirects(v)
+            )
+
+
 def test_valid_version() -> None:
     """Check that the package defines a valid __version__"""
     assert parse_version(__version__) >= parse_version("0.1.0")
@@ -32,7 +60,7 @@ def test_validate_doi() -> None:
             'https://linkinghub.elsevier.com/retrieve/pii/S0009261497040141'),
     ]
     for doi, url in data:
-        assert url == validate_doi(doi)
+        assert normalize_eq(url, validate_doi(doi))
 
     for doi in ['', 'asdf']:
         try:
@@ -49,7 +77,7 @@ def test_get_real_url_from_doi() -> None:
          'article/abs/pii/S0009261497040141'),
     ]
     for doi, url in data:
-        assert url == get_real_url_from_doi(doi)
+        assert normalize_eq(url, get_real_url_from_doi(doi))
 
 
 def test_find_doi_in_line() -> None: