ENH: Merge current versions from SciPy and NumPy

I think I got the most current versions of each

Author: DWesl

AUTHORS.rst

@@ -3,3 +3,9 @@ Contributors
 ============
 
 * DWesl <22566757+DWesl@users.noreply.github.com>
+* rgommers
+* nbelakovski
+* lucascolley
+* izaid
+* steppi
+* dschmitz

pyproject.toml

@@ -7,3 +7,6 @@ build-backend = "setuptools.build_meta"
 # For smarter version schemes and other configuration options,
 # check out https://github.com/pypa/setuptools_scm
 version_scheme = "no-guess-dev"
+
+[project.scripts]
+check_python_h_first = "check_python_h_first.__main__:main"

setup.cfg

@@ -106,7 +106,7 @@ formats = bdist_wheel
 [flake8]
 # Some sane defaults for the code style checker flake8
 max_line_length = 88
-extend_ignore = E203, W503
+extend_ignore = E203, W503, Q000
 # ^  Black-compatible
 #    E203 and W503 have edge cases handled by black
 exclude =
@@ -115,6 +115,19 @@ exclude =
     dist
     .eggs
     docs/conf.py
+# flake8-rst-docstrings
+rst-roles =
+    class,
+    func,
+    ref,
+rst-directives =
+    envvar,
+    exception,
+rst-substitutions =
+    version
+
+[pydocstyle]
+convention = numpy
 
 [pyscaffold]
 # PyScaffold's parameters when the project was created.

src/check_python_h_first/__main__.py

@@ -0,0 +1,36 @@
+"""Run the checker on the given files."""
+
+import argparse
+import os.path
+import sys
+
+from . import __version__
+from .wrapper import find_c_cpp_files, process_files
+
+PARSER = argparse.ArgumentParser(description=__doc__)
+PARSER.add_argument(
+    "files",
+    nargs="+",
+    help="Lint these files or this directory; use **/*.c to lint all files\n"
+    "Expects relative paths",
+)
+PARSER.add_argument(
+    "--version", action="version", version="check_python_h_first {__version__}"
+)
+
+
+def main():
+    """Run the checker on the files passed on the command line."""
+    args = PARSER.parse_args()
+
+    files = args.files
+    if len(files) == 1 and os.path.isdir(files[0]):
+        files = find_c_cpp_files(files[0])
+
+    # See which of the headers include Python.h and add them to the list
+    n_out_of_order = process_files(files)
+    sys.exit(n_out_of_order)
+
+
+if __name__ == "__main__":
+    main()

src/check_python_h_first/get_submodule_paths.py

@@ -0,0 +1,37 @@
+"""Find files in submodules.
+
+Those should be fixed in upstream project repos, not here.
+"""
+
+import glob
+import os.path
+
+
+def get_submodule_paths():
+    """Get submodule roots.
+
+    Get paths to submodules so that we can exclude them from things like
+    check_test_name.py, check_unicode.py, etc.
+    """
+    root_directory = os.path.dirname(os.path.dirname(__file__))
+    gitmodule_file = os.path.join(root_directory, ".gitmodules")
+    with open(gitmodule_file) as gitmodules:
+        data = gitmodules.read().split("\n")
+        submodule_paths = [
+            datum.split(" = ")[1] for datum in data if datum.startswith("\tpath = ")
+        ]
+        submodule_paths = [
+            os.path.join(root_directory, path) for path in submodule_paths
+        ]
+    # vendored with a script rather than via gitmodules
+    with open(os.path.join(root_directory, ".gitattributes"), "r") as attr_file:
+        for line in attr_file:
+            if "vendored" in line:
+                pattern = line.split(" ", 1)[0]
+                submodule_paths.extend(glob.glob(pattern))
+
+    return submodule_paths
+
+
+if __name__ == "__main__":
+    print("\n".join(get_submodule_paths()))

src/check_python_h_first/single_file.py

@@ -0,0 +1,132 @@
+"""Check that Python.h is included before any stdlib headers.
+
+May be a bit overzealous, but it should get the job done.
+"""
+
+import os.path
+import re
+import sys
+
+HEADER_PATTERN = re.compile(
+    r'^\s*#\s*include\s*[<"]((?:\w+/)*\w+(?:\.h[hp+]{0,2})?)[>"]\s*$'
+)
+
+PYTHON_INCLUDING_HEADERS = [
+    "Python.h",
+    # This isn't all of Python.h, but it is the visibility macros
+    "pyconfig.h",
+    # NumPy
+    "numpy/npy_common.h",
+    "numpy/npy_math.h",
+    "numpy/arrayobject.h",
+    "numpy/ndarrayobject.h",
+    "numpy/ndarraytypes.h",
+    "numpy/random/distributions.h",
+    # Pybind
+    "pybind11/pybind11.h",
+    # Boost::Python
+    "boost/python.hpp",
+    # Pythran
+    "pythonic/core.hpp",
+    # xsf::numpy
+    "xsf/numpy.h",
+]
+LEAF_HEADERS = [
+    "numpy/numpyconfig.h",
+    "numpy/npy_os.h",
+    "numpy/npy_cpu.h",
+    "numpy/utils.h",
+]
+
+
+def check_python_h_included_first(name_to_check: str) -> int:
+    """Check that the passed file includes Python.h first if it does at all.
+
+    Perhaps overzealous, but that should work around concerns with
+    recursion.
+
+    Parameters
+    ----------
+    name_to_check : str
+        The name of the file to check.
+
+    Returns
+    -------
+    int
+        The number of headers before Python.h
+    """
+    included_python = False
+    included_non_python_header = []
+    warned_python_construct = False
+    basename_to_check = os.path.basename(name_to_check)
+    in_comment = False
+    includes_headers = False
+    with open(name_to_check) as in_file:
+        for i, line in enumerate(in_file, 1):
+            # Very basic comment parsing
+            # Assumes /*...*/ comments are on their own lines
+            if "/*" in line:
+                if "*/" not in line:
+                    in_comment = True
+                # else-branch could use regex to remove comment and continue
+                continue
+            if in_comment:
+                if "*/" in line:
+                    in_comment = False
+                continue
+            line = line.split("//", 1)[0].strip()
+            # Now that there's no comments, look for headers
+            match = HEADER_PATTERN.match(line)
+            if match:
+                includes_headers = True
+                this_header = match.group(1)
+                if this_header in PYTHON_INCLUDING_HEADERS:
+                    if included_non_python_header and not included_python:
+                        # Headers before python-including header
+                        print(
+                            f"Header before Python.h in file {name_to_check:s}\n"
+                            f"Python.h on line {i:d}, other header(s) on line(s)"
+                            f" {included_non_python_header}",
+                            file=sys.stderr,
+                        )
+                    # else:  # no headers before python-including header
+                    included_python = True
+                    PYTHON_INCLUDING_HEADERS.append(basename_to_check)
+                    if os.path.dirname(name_to_check).endswith("include/numpy"):
+                        PYTHON_INCLUDING_HEADERS.append(f"numpy/{basename_to_check:s}")
+                    # We just found out where Python.h comes in this file
+                    break
+                elif this_header in LEAF_HEADERS:
+                    # This header is just defines, so it won't include
+                    # the system headers that cause problems
+                    continue
+                elif not included_python and (
+                    "numpy/" in this_header
+                    and this_header not in LEAF_HEADERS
+                    or "python" in this_header.lower()
+                    or "pybind" in this_header
+                ):
+                    print(
+                        f"Python.h not included before python-including header "
+                        f"in file {name_to_check:s}\n"
+                        f"{this_header:s} on line {i:d}",
+                        file=sys.stderr,
+                    )
+                    included_python = True
+                    PYTHON_INCLUDING_HEADERS.append(basename_to_check)
+                elif not included_python and this_header not in LEAF_HEADERS:
+                    included_non_python_header.append(i)
+            elif (
+                not included_python
+                and not warned_python_construct
+                and ".h" not in basename_to_check
+            ) and ("py::" in line or "PYBIND11_" in line):
+                print(
+                    "Python-including header not used before python constructs "
+                    f"in file {name_to_check:s}\nConstruct on line {i:d}",
+                    file=sys.stderr,
+                )
+                warned_python_construct = True
+    if not includes_headers:
+        LEAF_HEADERS.append(basename_to_check)
+    return included_python and len(included_non_python_header)