From 1b5c860c2a341e5da6c924ba16128ccc5158f193 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Tue, 9 Jan 2024 23:14:45 +0100 Subject: [PATCH 1/7] Remove distutils usage Backported using af37b3f28d61b4a0d04c64f87d170e6ddf3b2a8c and 799a104f42c88dd19d54ada4b5f8d26bff0169b1 * remove distutils usage * follow PEP632 * add own `strtobool` * use `new_compiler` from `setuptools.build_ext` for now * drop `SETUPTOOLS_USE_DISTUTILS=stdlib` * allow Cython 3 --- nrn_requirements.txt | 2 +- packaging/python/build_requirements.txt | 2 +- setup.cfg | 2 +- setup.py | 67 +++++++++++++------ .../neuron/rxd/geometry3d/CMakeLists.txt | 1 - .../python/neuron/rxd/geometry3d/setup.py.in | 7 +- .../lib/python/neuron/tests/utils/__init__.py | 0 .../python/neuron/tests/utils/strtobool.py | 16 +++++ share/lib/python/scripts/_binwrapper.py | 24 ++++++- src/neuronmusic/CMakeLists.txt | 35 +--------- src/neuronmusic/setup.py.in | 2 +- src/nrnpython/CMakeLists.txt | 38 +---------- src/nrnpython/setup.py.in | 30 +-------- test/coreneuron/test_datareturn.py | 6 +- test/coreneuron/test_direct.hoc | 2 +- test/coreneuron/test_direct.py | 6 +- test/coreneuron/test_fornetcon.py | 6 +- test/coreneuron/test_netmove.py | 6 +- test/coreneuron/test_psolve.py | 6 +- test/coreneuron/test_spikes.py | 16 ++--- test/coreneuron/test_units.py | 6 +- test/coreneuron/test_watchrange.py | 6 +- test/pynrn/test_fast_imem.py | 6 +- test/pynrn/test_version_macros.py | 4 +- 24 files changed, 122 insertions(+), 174 deletions(-) create mode 100644 share/lib/python/neuron/tests/utils/__init__.py create mode 100644 share/lib/python/neuron/tests/utils/strtobool.py diff --git a/nrn_requirements.txt b/nrn_requirements.txt index 972947f029..cddce9b9dc 100644 --- a/nrn_requirements.txt +++ b/nrn_requirements.txt @@ -5,7 +5,7 @@ matplotlib # bokeh 3 seems to break docs notebooks bokeh<3 ipython -cython<3 +cython packaging pytest pytest-cov diff --git a/packaging/python/build_requirements.txt b/packaging/python/build_requirements.txt index dcbd639f9c..ba9dcfb0e8 100644 --- a/packaging/python/build_requirements.txt +++ b/packaging/python/build_requirements.txt @@ -1,2 +1,2 @@ -cython<3 +cython packaging diff --git a/setup.cfg b/setup.cfg index c56e944fe9..54d0174845 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,7 @@ name = NEURON description = Empirically-based simulator for modeling neurons and networks of neurons author = Michael Hines, Yale, Blue Brain Project -author-email = michael.hines@yale.edu +author_email = michael.hines@yale.edu license = Copyright (c) Michael Hines (BSD compatible) url = https://neuron.yale.edu/neuron/ project_urls = diff --git a/setup.py b/setup.py index ef62207c3a..eea2c2dd62 100644 --- a/setup.py +++ b/setup.py @@ -4,9 +4,9 @@ import subprocess import sys from collections import defaultdict -from distutils import log -from distutils.dir_util import copy_tree -from distutils.version import LooseVersion +import logging +from shutil import copytree +from packaging.version import Version from setuptools import Command, Extension from setuptools import setup @@ -73,7 +73,7 @@ class Components: from Cython.Distutils import build_ext import numpy except ImportError: - log.error( + logging.error( "ERROR: RX3D wheel requires Cython and numpy. Please install beforehand" ) sys.exit(1) @@ -151,7 +151,7 @@ def run(self, *args, **kw): # Collect project files to be installed # These go directly into final package, regardless of setuptools filters - log.info("\n==> Collecting CMAKE files") + logging.info("\n==> Collecting CMAKE files") rel_package = ext.name.split(".")[:-1] package_data_d = os.path.join( self.build_lib, *(rel_package + [".data"]) @@ -162,16 +162,16 @@ def run(self, *args, **kw): ext.cmake_install_prefix, ext.cmake_install_python_files ) if os.path.isdir(src_py_dir): - copy_tree(src_py_dir, self.build_lib) # accepts existing dst dir + copytree(src_py_dir, self.build_lib, dirs_exist_ok=True) shutil.rmtree(src_py_dir) # avoid being collected to data dir for d in ext.cmake_collect_dirs: - log.info(" - Collecting %s (and everything under it)", d) + logging.info(" - Collecting %s (and everything under it)", d) src_dir = os.path.join(ext.cmake_install_prefix, d) dst_dir = os.path.join(package_data_d, d) if not os.path.isdir(dst_dir): shutil.copytree(src_dir, dst_dir) - log.info("==> Done building CMake project\n.") + logging.info("==> Done building CMake project\n.") # Make the temp include paths in the building the extension ext.include_dirs += [ @@ -182,7 +182,7 @@ def run(self, *args, **kw): ext.cmake_done = True # Now build the extensions normally - log.info("==> Building Python extensions") + logging.info("==> Building Python extensions") build_ext.run(self, *args, **kw) def _run_cmake(self, ext): @@ -190,7 +190,7 @@ def _run_cmake(self, ext): cfg = "Debug" if self.debug else "Release" self.outdir = os.path.abspath(ext.cmake_install_prefix) - log.info("Building lib to: %s", self.outdir) + logging.info("Building lib to: %s", self.outdir) cmake_args = [ # Generic options only. project options shall be passed as ext param "-DCMAKE_INSTALL_PREFIX=" + self.outdir, @@ -220,7 +220,9 @@ def _run_cmake(self, ext): try: # Configure project subprocess.Popen("echo $CXX", shell=True, stdout=subprocess.PIPE) - log.info("[CMAKE] cmd: %s", " ".join([cmake, ext.sourcedir] + cmake_args)) + logging.info( + "[CMAKE] cmd: %s", " ".join([cmake, ext.sourcedir] + cmake_args) + ) subprocess.check_call( [cmake, ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env ) @@ -279,7 +281,7 @@ def _run_cmake(self, ext): ) except subprocess.CalledProcessError as exc: - log.error("Status : FAIL. Log:\n%s", exc.output) + logging.error("Status : FAIL. Logging.\n%s", exc.output) raise @staticmethod @@ -287,10 +289,10 @@ def _find_cmake(): for candidate in ["cmake", "cmake3"]: try: out = subprocess.check_output([candidate, "--version"]) - cmake_version = LooseVersion( + cmake_version = Version( re.search(r"version\s*([\d.]+)", out.decode()).group(1) ) - if cmake_version >= "3.15.0": + if cmake_version >= Version("3.15.0"): return candidate except OSError: pass @@ -331,6 +333,7 @@ def setup_package(): "neuron", "neuron.neuroml", "neuron.tests", + "neuron.tests.utils", "neuron.rxd", "neuron.crxd", "neuron.gui2", @@ -406,7 +409,7 @@ def setup_package(): ) ) - log.info("RX3D compile flags %s" % str(rxd_params)) + logging.info("RX3D compile flags %s" % str(rxd_params)) extensions += [ CyExtension( @@ -432,7 +435,7 @@ def setup_package(): ), ] - log.info("RX3D is %s", "ENABLED" if Components.RX3D else "DISABLED") + logging.info("RX3D is %s", "ENABLED" if Components.RX3D else "DISABLED") # package name package_name = "NEURON-gpu" if Components.GPU else "NEURON" @@ -480,7 +483,7 @@ def mac_osx_setenv(): .decode() .strip() ) - log.info("Setting SDKROOT=%s", sdk_root) + logging.info("Setting SDKROOT=%s", sdk_root) os.environ["SDKROOT"] = sdk_root # Match Python OSX framework @@ -493,9 +496,33 @@ def mac_osx_setenv(): " for a recent MACOS version (from brew?). Your wheel won't be portable." " Consider using an official Python build from python.org" ) - macos_target = "%d.%d" % tuple(py_osx_framework[:2]) - log.info("Setting MACOSX_DEPLOYMENT_TARGET=%s", macos_target) - os.environ["MACOSX_DEPLOYMENT_TARGET"] = macos_target + if py_osx_framework is not None and explicit_target > py_osx_framework: + logging.warn( + "You are building wheels for macOS >={}; this is more " + "restrictive than your Python framework, which supports " + ">={}".format(fmt(explicit_target), fmt(py_osx_framework)) + ) + else: + # Target not set explicitly, set MACOSX_DEPLOYMENT_TARGET to match the + # Python framework, or 10.9 if the version targeted by the framework + # cannot be determined + if py_osx_framework is None: + py_osx_framework = (10, 15) + if py_osx_framework < (10, 15): + logging.warn( + "C++17 support is required to build NEURON on macOS, " + "therefore minimum MACOSX_DEPLOYMENT_TARGET version is 10.15." + ) + py_osx_framework = (10, 15) + if py_osx_framework > (10, 15): + logging.warn( + "You are building a wheel with a Python built for macOS >={}. " + "Your wheel won't run on older versions, consider using an " + "official Python build from python.org".format(fmt(py_osx_framework)) + ) + macos_target = "%d.%d" % tuple(py_osx_framework[:2]) + logging.warn("Setting MACOSX_DEPLOYMENT_TARGET=%s", macos_target) + os.environ["MACOSX_DEPLOYMENT_TARGET"] = macos_target if __name__ == "__main__": diff --git a/share/lib/python/neuron/rxd/geometry3d/CMakeLists.txt b/share/lib/python/neuron/rxd/geometry3d/CMakeLists.txt index afcfa77f8d..2a298da327 100644 --- a/share/lib/python/neuron/rxd/geometry3d/CMakeLists.txt +++ b/share/lib/python/neuron/rxd/geometry3d/CMakeLists.txt @@ -70,7 +70,6 @@ mingw=${MINGW}\n\ shift\n\ export LDCSHARED=\"${CMAKE_C_COMPILER} ${CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS}\"\n\ export LDCXXSHARED=\"${CMAKE_CXX_COMPILER} ${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS}\"\n\ -export SETUPTOOLS_USE_DISTUTILS=stdlib if test x$mingw = x1 ; then\n\ pyver=`$pyexe -c 'import sys; print (sys.version_info[0]); quit()'`\n\ echo pyver=$pyver\n\ diff --git a/share/lib/python/neuron/rxd/geometry3d/setup.py.in b/share/lib/python/neuron/rxd/geometry3d/setup.py.in index 4a51315c70..82541c76f0 100644 --- a/share/lib/python/neuron/rxd/geometry3d/setup.py.in +++ b/share/lib/python/neuron/rxd/geometry3d/setup.py.in @@ -16,16 +16,15 @@ else: # not an absolute path pgi_compiler_flags = "-noswitcherror" -from distutils.core import setup -from distutils.extension import Extension +from setuptools import setup, Extension def have_vc(): if not mingw: return False import traceback try: - from distutils import spawn - x = spawn.find_executable("cl") + from shutil import which + x = which("cl") x = True if x is not None and "Microsoft" in x else False except: traceback.print_exc() diff --git a/share/lib/python/neuron/tests/utils/__init__.py b/share/lib/python/neuron/tests/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/share/lib/python/neuron/tests/utils/strtobool.py b/share/lib/python/neuron/tests/utils/strtobool.py new file mode 100644 index 0000000000..d5e04a7df8 --- /dev/null +++ b/share/lib/python/neuron/tests/utils/strtobool.py @@ -0,0 +1,16 @@ +# Own implementation of distutils.util.strtobool +# See PEP632 for more info. +def strtobool(val) -> bool: + """Convert a string representation of truth to True or False. + + True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values + are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if + 'val' is anything else. + """ + val = val.lower() + if val in ("y", "yes", "t", "true", "on", "1"): + return True + elif val in ("n", "no", "f", "false", "off", "0"): + return False + else: + raise ValueError("invalid truth value %r" % (val,)) diff --git a/share/lib/python/scripts/_binwrapper.py b/share/lib/python/scripts/_binwrapper.py index 0ecfb8177b..f1e2e50d4e 100755 --- a/share/lib/python/scripts/_binwrapper.py +++ b/share/lib/python/scripts/_binwrapper.py @@ -8,11 +8,11 @@ import subprocess import sys from pkg_resources import working_set -from distutils.ccompiler import new_compiler +from setuptools.command.build_ext import new_compiler +from packaging.version import Version from sysconfig import get_config_vars, get_config_var -# This replaces the now depricated distutils.sysutils.customize_compiler def _customize_compiler(compiler): """Do platform-sepcific customizations of compilers on unix platforms.""" if compiler.compiler_type == "unix": @@ -40,6 +40,26 @@ def _set_default_compiler(): os.environ.setdefault("CXX", ccompiler.compiler_cxx[0]) +def _check_cpp_compiler_version(): + """Check if GCC compiler is >= 9.0 otherwise show warning""" + try: + cpp_compiler = os.environ.get("CXX", "") + version = subprocess.run( + [cpp_compiler, "--version"], stdout=subprocess.PIPE + ).stdout.decode("utf-8") + if "GCC" in version: + version = subprocess.run( + [cpp_compiler, "-dumpversion"], stdout=subprocess.PIPE + ).stdout.decode("utf-8") + if Version(version) <= Version("9.0"): + print( + "Warning: GCC >= 9.0 is required with this version of NEURON but found", + version, + ) + except: + pass + + def _config_exe(exe_name): """Sets the environment to run the real executable (returned)""" diff --git a/src/neuronmusic/CMakeLists.txt b/src/neuronmusic/CMakeLists.txt index 7486ffeae6..e83c8d1d5e 100644 --- a/src/neuronmusic/CMakeLists.txt +++ b/src/neuronmusic/CMakeLists.txt @@ -26,7 +26,6 @@ mingw=${MINGW}\n\ shift\n\ export LDCSHARED=\"${CMAKE_C_COMPILER} ${CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS}\"\n\ export LDCXXSHARED=\"${CMAKE_CXX_COMPILER} ${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS}\"\n\ -export SETUPTOOLS_USE_DISTUTILS=stdlib if test x$mingw = x1 ; then\n\ pyver=`$pyexe -c 'import sys; print (sys.version_info[0]); quit()'`\n\ echo pyver=$pyver\n\ @@ -82,37 +81,7 @@ fi\n\ endforeach(pyexe) add_dependencies(neuronmusicextension nrniv_lib neuronmusic_cython_generated) - - # ~~~ - # (Copied from src/nrnpython/CMakeLists.txt) - # neuron module (possibly with multiple extension versions) was built - # in NRN_PYTHON_BUILD_LIB. Not a problem if install overwrites multiple - # times to same install folder or if each install ends up in different - # place. - # ~~~ - file( - WRITE ${CMAKE_CURRENT_BINARY_DIR}/neuronmusic_module_install.sh - "\ -#!bash\n\ -echo 'Installing python module using:'\n\ -set -ex\n\ -cd ${CMAKE_CURRENT_BINARY_DIR}\n\ -export SETUPTOOLS_USE_DISTUTILS=stdlib -$1 setup.py --quiet build --build-lib=${NRN_PYTHON_BUILD_LIB} install ${NRN_MODULE_INSTALL_OPTIONS}\n\ -") - foreach(pyexe ${NRN_PYTHON_EXE_LIST}) - # install(CODE ...) only takes a single CMake code expression, so we can't easily roll our own - # check on the return code. Modern CMake versions support the COMMAND_ERROR_IS_FATAL option, - # which will cause the installation to abort if the shell script returns an error code. - if(${CMAKE_VERSION} VERSION_LESS "3.19") - install( - CODE "execute_process(COMMAND bash ${CMAKE_CURRENT_BINARY_DIR}/neuronmusic_module_install.sh ${pyexe})" - ) - else() - install( - CODE "execute_process(COMMAND bash ${CMAKE_CURRENT_BINARY_DIR}/neuronmusic_module_install.sh ${pyexe} COMMAND_ERROR_IS_FATAL LAST)" - ) - endif() - endforeach(pyexe) + # install modules from NRN_PYTHON_BUILD_LIB + install(DIRECTORY ${NRN_PYTHON_BUILD_LIB} DESTINATION lib) endif() diff --git a/src/neuronmusic/setup.py.in b/src/neuronmusic/setup.py.in index c4bd985f40..3d7c129121 100644 --- a/src/neuronmusic/setup.py.in +++ b/src/neuronmusic/setup.py.in @@ -1,5 +1,5 @@ #setup.py for neuronmusic extension -from distutils.core import setup, Extension +from setuptools import setup, Extension nrn_srcdir = "@NRN_SRCDIR@" instdir = "@prefix@" diff --git a/src/nrnpython/CMakeLists.txt b/src/nrnpython/CMakeLists.txt index e4a4e3f187..8863183580 100644 --- a/src/nrnpython/CMakeLists.txt +++ b/src/nrnpython/CMakeLists.txt @@ -220,11 +220,6 @@ if(NRN_ENABLE_MODULE_INSTALL) # ============================================================================= # for each python detected / provided by user, install module at install time - # Workaround for: https://github.com/neuronsimulator/nrn/issues/1605 See: - # https://setuptools.pypa.io/en/latest/deprecated/distutils-legacy.html - # https://setuptools.pypa.io/en/latest/history.html#id228 - set(extra_env ${CMAKE_COMMAND} -E env "SETUPTOOLS_USE_DISTUTILS=stdlib") - if(NRN_SANITIZERS) # Make sure the same compiler (currently always clang in the sanitizer builds) is used to link # as was used to build. By default, Python will try to use the compiler that was used to build @@ -252,35 +247,6 @@ if(NRN_ENABLE_MODULE_INSTALL) if(NRN_ENABLE_RX3D) add_dependencies(hoc_module rxd_cython_generated) endif() - - # ~~~ - # neuron module (possibly with multiple extension versions) was built - # in NRN_PYTHON_BUILD_LIB. Not a problem if install overwrites multiple - # times to same install folder or if each install ends up in different - # place. - # ~~~ - file( - WRITE ${CMAKE_CURRENT_BINARY_DIR}/neuron_module_install.sh - "\ -#!bash\n\ -echo 'Installing python module using:'\n\ -set -ex\n\ -cd ${CMAKE_CURRENT_BINARY_DIR}\n\ -export SETUPTOOLS_USE_DISTUTILS=stdlib -$1 setup.py --quiet build --build-lib=${NRN_PYTHON_BUILD_LIB} install ${NRN_MODULE_INSTALL_OPTIONS}\n\ -") - foreach(pyexe ${NRN_PYTHON_EXE_LIST}) - # install(CODE ...) only takes a single CMake code expression, so we can't easily roll our own - # check on the return code. Modern CMake versions support the COMMAND_ERROR_IS_FATAL option, - # which will cause the installation to abort if the shell script returns an error code. - if(${CMAKE_VERSION} VERSION_LESS "3.19") - install( - CODE "execute_process(COMMAND bash ${CMAKE_CURRENT_BINARY_DIR}/neuron_module_install.sh ${pyexe})" - ) - else() - install( - CODE "execute_process(COMMAND bash ${CMAKE_CURRENT_BINARY_DIR}/neuron_module_install.sh ${pyexe} COMMAND_ERROR_IS_FATAL LAST)" - ) - endif() - endforeach(pyexe) + # install modules from NRN_PYTHON_BUILD_LIB + install(DIRECTORY ${NRN_PYTHON_BUILD_LIB} DESTINATION lib) endif() diff --git a/src/nrnpython/setup.py.in b/src/nrnpython/setup.py.in index 0c8f325a91..4c38189c0c 100644 --- a/src/nrnpython/setup.py.in +++ b/src/nrnpython/setup.py.in @@ -1,5 +1,5 @@ #setup.py -from distutils.core import setup, Extension +from setuptools import setup, Extension from sysconfig import get_python_version import sys @@ -82,26 +82,6 @@ if using_pgi: os.environ["LDSHARED"] = os.environ['CC'] + " -shared" os.environ["LDCXXSHARED"] = os.environ["CXX"] + " -shared" -# apparently we do not need the following -################################# -## following http://code.google.com/p/maroonmpi/wiki/Installation -## hack into distutils to replace the compiler in "linker_so" with mpicxx_bin -# -#import distutils -#import distutils.unixccompiler -# -#class MPI_UnixCCompiler(distutils.unixccompiler.UnixCCompiler): -# __set_executable = distutils.unixccompiler.UnixCCompiler.set_executable -# -# def set_executable(self,key,value): -# print "MPI_UnixCCompiler ", key, " | ", value -# if key == 'linker_so' and type(value) == str: -# value = mpicxx_bin + ' ' + ' '.join(value.split()[1:]) -# -# return self.__set_executable(key,value) -# -#distutils.unixccompiler.UnixCCompiler = MPI_UnixCCompiler -################################# #include_dirs for hoc module include_dirs = [nrn_srcdir+'/src/nrnpython', nrn_srcdir+'/src/oc', '../oc', nrn_srcdir+'/src/nrnmpi'] @@ -161,12 +141,6 @@ hoc_module = Extension( define_macros=defines ) -# specify that the data_files paths are relative to same place as python files -# from http://stackoverflow.com/questions/1612733/including-non-python-files-with-setup-py -from distutils.command.install import INSTALL_SCHEMES -for scheme in list(INSTALL_SCHEMES.values()): - scheme['data'] = scheme['purelib'] - ext_modules = [hoc_module] # The rx3d extensions are built using the setup.py.in in @@ -214,7 +188,7 @@ if build_rx3d: except: pass -packages=['neuron','neuron.neuroml','neuron.tests', 'neuron.rxd', 'neuron.crxd', 'neuron.gui2'] +packages=['neuron','neuron.neuroml','neuron.tests', 'neuron.tests.utils', 'neuron.rxd', 'neuron.crxd', 'neuron.gui2'] if build_rx3d: packages +=['neuron.rxd.geometry3d'] diff --git a/test/coreneuron/test_datareturn.py b/test/coreneuron/test_datareturn.py index aea3b05067..a634c63712 100644 --- a/test/coreneuron/test_datareturn.py +++ b/test/coreneuron/test_datareturn.py @@ -1,5 +1,5 @@ # Test of data return covering most of the functionality. -import distutils.util +from neuron.tests.utils.strtobool import strtobool import itertools import os @@ -175,9 +175,7 @@ def run(tstop, mode): h.CVode().cache_efficient(1) coreneuron.enable = True coreneuron.verbose = 0 - coreneuron.gpu = bool( - distutils.util.strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false")) - ) + coreneuron.gpu = bool(strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false"))) results = [] cell_permute_values = (1, 2) if coreneuron.gpu else (0, 1) diff --git a/test/coreneuron/test_direct.hoc b/test/coreneuron/test_direct.hoc index 5700f88953..4a524fc10e 100644 --- a/test/coreneuron/test_direct.hoc +++ b/test/coreneuron/test_direct.hoc @@ -53,7 +53,7 @@ proc test_direct_memory_transfer() { localobj po, pc, ic, tv, vvec, i_mem, tvstd po = new PythonObject() po.coreneuron.enable = 1 - nrnpython("import distutils.util; import os; coreneuron.gpu=bool(distutils.util.strtobool(os.environ.get('CORENRN_ENABLE_GPU', 'false')))") + nrnpython("from neuron.tests.utils.strtobool import strtobool; import os; coreneuron.gpu=bool(strtobool(os.environ.get('CORENRN_ENABLE_GPU', 'false')))") printf("nrncore_arg: |%s|\n", po.coreneuron.nrncore_arg(tstop)) pc = new ParallelContext() diff --git a/test/coreneuron/test_direct.py b/test/coreneuron/test_direct.py index 8f504d4377..cbb37f8405 100644 --- a/test/coreneuron/test_direct.py +++ b/test/coreneuron/test_direct.py @@ -1,4 +1,4 @@ -import distutils.util +from neuron.tests.utils.strtobool import strtobool import os from neuron import h, gui @@ -38,9 +38,7 @@ def test_direct_memory_transfer(): coreneuron.enable = True coreneuron.verbose = 0 coreneuron.model_stats = True - coreneuron.gpu = bool( - distutils.util.strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false")) - ) + coreneuron.gpu = bool(strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false"))) coreneuron.num_gpus = 1 pc = h.ParallelContext() diff --git a/test/coreneuron/test_fornetcon.py b/test/coreneuron/test_fornetcon.py index cd063ed7f8..e668da5dfe 100644 --- a/test/coreneuron/test_fornetcon.py +++ b/test/coreneuron/test_fornetcon.py @@ -1,7 +1,7 @@ # Basically want to test that FOR_NETCONS statement works when # the NetCons connecting to ForNetConTest instances are created # in random order. -import distutils.util +from neuron.tests.utils.strtobool import strtobool import os from neuron import h @@ -85,9 +85,7 @@ def get_weights(): print("CoreNEURON run") h.CVode().cache_efficient(1) coreneuron.enable = True - coreneuron.gpu = bool( - distutils.util.strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false")) - ) + coreneuron.gpu = bool(strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false"))) def runassert(mode): spiketime.resize(0) diff --git a/test/coreneuron/test_netmove.py b/test/coreneuron/test_netmove.py index 5ced8fa76c..32d3539e67 100644 --- a/test/coreneuron/test_netmove.py +++ b/test/coreneuron/test_netmove.py @@ -1,6 +1,6 @@ # Basically want to test that net_move statement doesn't get # mixed up with other instances. -import distutils.util +from neuron.tests.utils.strtobool import strtobool import os from neuron import h @@ -79,9 +79,7 @@ def run(tstop, mode): h.CVode().cache_efficient(1) coreneuron.enable = True coreneuron.verbose = 0 - coreneuron.gpu = bool( - distutils.util.strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false")) - ) + coreneuron.gpu = bool(strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false"))) def runassert(mode): run(tstop, mode) diff --git a/test/coreneuron/test_psolve.py b/test/coreneuron/test_psolve.py index bef9ff95c7..0411160e05 100644 --- a/test/coreneuron/test_psolve.py +++ b/test/coreneuron/test_psolve.py @@ -1,4 +1,4 @@ -import distutils.util +from neuron.tests.utils.strtobool import strtobool import os from neuron import h, gui @@ -47,9 +47,7 @@ def run(tstop): coreneuron.enable = True coreneuron.verbose = 0 - coreneuron.gpu = bool( - distutils.util.strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false")) - ) + coreneuron.gpu = bool(strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false"))) h.CVode().cache_efficient(True) run(h.tstop) if vvec_std.eq(vvec) == 0: diff --git a/test/coreneuron/test_spikes.py b/test/coreneuron/test_spikes.py index 4fdce9a75e..677cffe81d 100644 --- a/test/coreneuron/test_spikes.py +++ b/test/coreneuron/test_spikes.py @@ -1,18 +1,12 @@ -import distutils.util +from neuron.tests.utils.strtobool import strtobool import os # Hacky, but it's non-trivial to pass commandline arguments to pytest tests. -enable_gpu = bool( - distutils.util.strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false")) -) -mpi4py_option = bool( - distutils.util.strtobool(os.environ.get("NRN_TEST_SPIKES_MPI4PY", "false")) -) -file_mode_option = bool( - distutils.util.strtobool(os.environ.get("NRN_TEST_SPIKES_FILE_MODE", "false")) -) +enable_gpu = bool(strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false"))) +mpi4py_option = bool(strtobool(os.environ.get("NRN_TEST_SPIKES_MPI4PY", "false"))) +file_mode_option = bool(strtobool(os.environ.get("NRN_TEST_SPIKES_FILE_MODE", "false"))) nrnmpi_init_option = bool( - distutils.util.strtobool(os.environ.get("NRN_TEST_SPIKES_NRNMPI_INIT", "false")) + strtobool(os.environ.get("NRN_TEST_SPIKES_NRNMPI_INIT", "false")) ) # following at top level and early enough avoids... diff --git a/test/coreneuron/test_units.py b/test/coreneuron/test_units.py index 6d89b5f697..cd35abcb47 100644 --- a/test/coreneuron/test_units.py +++ b/test/coreneuron/test_units.py @@ -1,4 +1,4 @@ -import distutils.util +from neuron.tests.utils.strtobool import strtobool import os from neuron import h @@ -20,9 +20,7 @@ def test_units(): h.CVode().cache_efficient(1) coreneuron.enable = True - coreneuron.gpu = bool( - distutils.util.strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false")) - ) + coreneuron.gpu = bool(strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false"))) pc.set_maxstep(10) h.finitialize(-65) pc.psolve(h.dt) diff --git a/test/coreneuron/test_watchrange.py b/test/coreneuron/test_watchrange.py index 0f8b3d81d2..fe5bfce203 100644 --- a/test/coreneuron/test_watchrange.py +++ b/test/coreneuron/test_watchrange.py @@ -1,6 +1,6 @@ # Basically want to test that net_move statement doesn't get # mixed up with other instances. -import distutils.util +from neuron.tests.utils.strtobool import strtobool import os from neuron import h @@ -91,9 +91,7 @@ def run(tstop, mode): h.CVode().cache_efficient(1) coreneuron.enable = True coreneuron.verbose = 0 - coreneuron.gpu = bool( - distutils.util.strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false")) - ) + coreneuron.gpu = bool(strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false"))) def runassert(mode): run(tstop, mode) diff --git a/test/pynrn/test_fast_imem.py b/test/pynrn/test_fast_imem.py index 74d7bd4f38..564737cc02 100644 --- a/test/pynrn/test_fast_imem.py +++ b/test/pynrn/test_fast_imem.py @@ -1,7 +1,7 @@ # Sum of all i_membrane_ should equal sum of all ElectrodeCurrent # For a demanding test, use a tree with many IClamp and ExpSyn point processes # sprinkled on zero and non-zero area nodes. -import distutils.util +from neuron.tests.utils.strtobool import strtobool import os from neuron import h @@ -310,9 +310,7 @@ def compare(): coreneuron.enable = True coreneuron.verbose = 0 - coreneuron.gpu = distutils.util.strtobool( - os.environ.get("CORENRN_ENABLE_GPU", "false") - ) + coreneuron.gpu = strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false")) run(tstop) compare() coreneuron.enable = False diff --git a/test/pynrn/test_version_macros.py b/test/pynrn/test_version_macros.py index 578fc2eeb3..f2374a91dd 100644 --- a/test/pynrn/test_version_macros.py +++ b/test/pynrn/test_version_macros.py @@ -1,4 +1,4 @@ -import distutils.util +from neuron.tests.utils.strtobool import strtobool import os from neuron import coreneuron, h @@ -17,7 +17,7 @@ def test_version_macros(): s = h.Section() s.insert("VersionMacros") coreneuron.enable = bool( - distutils.util.strtobool(os.environ.get("NRN_CORENEURON_ENABLE", "false")) + strtobool(os.environ.get("NRN_CORENEURON_ENABLE", "false")) ) coreneuron.verbose = True h.CVode().cache_efficient(True) From e42fa537226769d6bdeabf23787a109bbf0ce3ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandru=20S=C4=83vulescu?= Date: Wed, 8 Mar 2023 11:53:39 +0100 Subject: [PATCH 2/7] one setup.py to rule them all (#2235) * one setup.py to rule them all * WinCI: also install Cython * -noswitcherror * handle flags * fmt <3 * RX3D Optimization level handling hoc_module depends on rxdmath * a bit of cleanup * Address CR * --force Cython rebuid in case of PYTHON_DYNAMIC * update packaging handling * cmake_min_version already handled in CMake * drop check from setup.py * fmt <3 * Update setup.py * fix PGI/NVHPC build regression --- CMakeLists.txt | 16 +- ci/win_install_deps.cmd | 10 +- cmake/CompilerHelper.cmake | 6 +- cmake/ConfigFileSetting.cmake | 12 +- mingw_files/vcenv.sh | 11 - setup.py | 174 +++++++++++---- .../neuron/rxd/geometry3d/CMakeLists.txt | 135 ------------ .../python/neuron/rxd/geometry3d/setup.py.in | 92 -------- src/neuronmusic/CMakeLists.txt | 87 -------- src/nrnpython/CMakeLists.txt | 152 ++++++------- src/nrnpython/setup.py.in | 202 ------------------ 11 files changed, 214 insertions(+), 683 deletions(-) delete mode 100644 mingw_files/vcenv.sh delete mode 100644 share/lib/python/neuron/rxd/geometry3d/CMakeLists.txt delete mode 100644 share/lib/python/neuron/rxd/geometry3d/setup.py.in delete mode 100644 src/neuronmusic/CMakeLists.txt delete mode 100644 src/nrnpython/setup.py.in diff --git a/CMakeLists.txt b/CMakeLists.txt index a587733139..f087952f18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -98,9 +98,8 @@ mark_as_advanced(NRN_WHEEL_BUILD) # ~~~ # NEURON module installation: # - OFF : do not install -# - ON : install with --home in ${CMAKE_INSTALL_PREFIX} -# - : install using other modes or locations using an appropriate -# string that goes after python setup.py install +# - ON : install in ${CMAKE_INSTALL_PREFIX} (default) +# NOTE: When building the wheel, this is set to OFF. # Dynamic Python version support: # - OFF : nrnpython interface is linked into libnrniv.so # - ON : nrnpython interface consistent with default python3 (falling back to python) @@ -124,7 +123,8 @@ mark_as_advanced(NRN_WHEEL_BUILD) # - ON : Collect code coverage for files (default all). # - : semicolon (;) separated list of files to collect coverage. # ~~~ -option(NRN_ENABLE_MODULE_INSTALL "Enable installation of NEURON Python module" +option(NRN_ENABLE_MODULE_INSTALL + "Enable installation of NEURON Python module for regular CMake builds" ${NRN_ENABLE_MODULE_INSTALL_DEFAULT}) set(NRN_MODULE_INSTALL_OPTIONS "${NRN_MODULE_INSTALL_OPTIONS_DEFAULT}" @@ -586,14 +586,6 @@ if(NRN_ENABLE_PYTHON) add_subdirectory(src/nrnpython) endif() -if(NRN_ENABLE_RX3D) - add_subdirectory(share/lib/python/neuron/rxd/geometry3d) -endif() - -if(NRN_ENABLE_MUSIC) - add_subdirectory(src/neuronmusic) -endif() - if(NRN_MACOS_BUILD) add_subdirectory(src/mac) endif() diff --git a/ci/win_install_deps.cmd b/ci/win_install_deps.cmd index 4b8f24297e..a8691b7a86 100644 --- a/ci/win_install_deps.cmd +++ b/ci/win_install_deps.cmd @@ -25,11 +25,10 @@ pwsh -command "(Get-Content C:\Python310\Lib\distutils\cygwinccompiler.py) -repl pwsh -command "(Get-Content C:\Python311\Lib\distutils\cygwinccompiler.py) -replace 'msvcr100', 'msvcrt' | Out-File C:\Python311\Lib\distutils\cygwinccompiler.py" :: install numpy -C:\Python37\python.exe -m pip install numpy==1.14.6 || goto :error -C:\Python38\python.exe -m pip install numpy==1.17.5 || goto :error -C:\Python39\python.exe -m pip install numpy==1.19.3 || goto :error -C:\Python310\python.exe -m pip install numpy==1.21.3 || goto :error -C:\Python311\python.exe -m pip install numpy==1.23.5 || goto :error +C:\Python38\python.exe -m pip install numpy==1.17.5 cython || goto :error +C:\Python39\python.exe -m pip install numpy==1.19.3 cython || goto :error +C:\Python310\python.exe -m pip install numpy==1.21.3 cython || goto :error +C:\Python311\python.exe -m pip install numpy==1.23.5 cython || goto :error :: install nsis nsis-3.05-setup.exe /S || goto :error @@ -66,6 +65,7 @@ mingw-w64-x86_64-readline ^ mingw-w64-x86_64-python3 ^ mingw64/mingw-w64-x86_64-cython ^ mingw-w64-x86_64-python3-setuptools ^ +mingw-w64-x86_64-python3-packaging ^ mingw-w64-x86_64-python3-pip ^ mingw64/mingw-w64-x86_64-dlfcn ^ mingw-w64-x86_64-toolchain || goto :error diff --git a/cmake/CompilerHelper.cmake b/cmake/CompilerHelper.cmake index 422a15960e..886b8d411c 100644 --- a/cmake/CompilerHelper.cmake +++ b/cmake/CompilerHelper.cmake @@ -11,8 +11,6 @@ endif() if(CMAKE_C_COMPILER_ID MATCHES "PGI" OR CMAKE_C_COMPILER_ID MATCHES "NVHPC") set(NRN_HAVE_NVHPC_COMPILER ON) - set(USING_PGI_COMPILER_TRUE "") - set(USING_PGI_COMPILER_FALSE "#") # See https://gitlab.kitware.com/cmake/cmake/-/issues/22168, upper limit of 3.20.3 is based on the # current assigned milestone there. if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.20" AND ${CMAKE_VERSION} VERSION_LESS "3.20.3") @@ -43,6 +41,8 @@ if(CMAKE_C_COMPILER_ID MATCHES "PGI" OR CMAKE_C_COMPILER_ID MATCHES "NVHPC") list(APPEND NRN_COMPILE_FLAGS --diag_suppress=1,47,111,128,170,174,177,180,186,301,541,550,816,941,2465) endif() + list(APPEND NRN_COMPILE_FLAGS -noswitcherror) + list(APPEND NRN_LINK_FLAGS -noswitcherror) if(${CMAKE_C_COMPILER_VERSION} VERSION_GREATER_EQUAL 21.11) # Random123 does not play nicely with NVHPC 21.11+'s detection of ABM features, see: # https://github.com/BlueBrain/CoreNeuron/issues/724 and @@ -51,8 +51,6 @@ if(CMAKE_C_COMPILER_ID MATCHES "PGI" OR CMAKE_C_COMPILER_ID MATCHES "NVHPC") endif() else() set(NRN_HAVE_NVHPC_COMPILER OFF) - set(USING_PGI_COMPILER_TRUE "#") - set(USING_PGI_COMPILER_FALSE "") endif() if(CMAKE_CXX_COMPILER_ID MATCHES "PGI") diff --git a/cmake/ConfigFileSetting.cmake b/cmake/ConfigFileSetting.cmake index 4fc71e8421..bfa9d5001f 100644 --- a/cmake/ConfigFileSetting.cmake +++ b/cmake/ConfigFileSetting.cmake @@ -310,12 +310,6 @@ else() file(REMOVE "${PROJECT_BINARY_DIR}/config.h") endif() -# Prepare some variables for @VAR@ expansion in setup.py.in (nrnpython and rx3d) -set(NRN_COMPILE_FLAGS_QUOTED ${NRN_COMPILE_FLAGS}) -set(NRN_LINK_FLAGS_QUOTED ${NRN_LINK_FLAGS}) -list(TRANSFORM NRN_COMPILE_FLAGS_QUOTED APPEND "'") -list(TRANSFORM NRN_COMPILE_FLAGS_QUOTED PREPEND "'") -list(TRANSFORM NRN_LINK_FLAGS_QUOTED APPEND "'") -list(TRANSFORM NRN_LINK_FLAGS_QUOTED PREPEND "'") -string(JOIN ", " NRN_COMPILE_FLAGS_COMMA_SEPARATED_STRINGS ${NRN_COMPILE_FLAGS_QUOTED}) -string(JOIN ", " NRN_LINK_FLAGS_COMMA_SEPARATED_STRINGS ${NRN_LINK_FLAGS_QUOTED}) +# Prepare some variables for setup.py extension building (hoc_module, rx3d and music) +string(JOIN " " NRN_COMPILE_FLAGS_STRING ${NRN_COMPILE_FLAGS}) +string(JOIN " " NRN_LINK_FLAGS_STRING ${NRN_LINK_FLAGS}) diff --git a/mingw_files/vcenv.sh b/mingw_files/vcenv.sh deleted file mode 100644 index a7dfe0f179..0000000000 --- a/mingw_files/vcenv.sh +++ /dev/null @@ -1,11 +0,0 @@ -export UCRTContentRoot='C:\Program Files (x86)\Windows Kits\10\' -export INCLUDE='C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\include;C:\Program Files (x86)\Windows Kits\10\Include\10.0.10240.0\ucrt;' -export VCTargetsPath='C:\Program Files (x86)\MSBuild\Microsoft.Cpp\v4.0\v140\' -export Platform='X64' -export VCINSTALLDIR='C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\' -export LIB='C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\lib\amd64;C:\Program Files (x86)\Windows Kits\10\lib\10.0.10240.0\ucrt\x64;' -export LIBPATH='C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\lib\amd64;' -export WindowsSdkDir='C:\Program Files (x86)\Windows Kits\10\' -export UCRTVersion='10.0.10240.0' -export UniversalCRTSdkDir='C:\Program Files (x86)\Windows Kits\10' -export PATH='/c/Program Files (x86)/Microsoft Visual Studio 14.0/VC/bin/amd64:/c/Program Files (x86)/Windows Kits/10/bin/x64:/c/Program Files (x86)/Windows Kits/10/bin/x86':$PATH diff --git a/setup.py b/setup.py index eea2c2dd62..e4bcfa0753 100644 --- a/setup.py +++ b/setup.py @@ -5,25 +5,76 @@ import sys from collections import defaultdict import logging -from shutil import copytree -from packaging.version import Version + +logging.basicConfig(level=logging.INFO) +from shutil import copytree, which from setuptools import Command, Extension from setuptools import setup +logging.info("setup.py called with:" + " ".join(sys.argv)) + + class Components: RX3D = True IV = True MPI = True + MUSIC = False # still early support CORENRN = False # still early support GPU = False # still early support -if os.name != "posix": - raise Exception( - "Python NEURON distributions are currently only available " - "for Mac and Linux systems (POSIX)" - ) +# Check if we've got --cmake-build-dir path that will be used to build extensions only +# and not to build NEURON wheels. +just_extensions = False +cmake_build_dir = "build/cmake_install" # default for wheels +if "--cmake-build-dir" in sys.argv: + cmake_build_dir = sys.argv[sys.argv.index("--cmake-build-dir") + 1] + sys.argv.remove("--cmake-build-dir") + sys.argv.remove(cmake_build_dir) + # Check that the CMake build directory is valid + if not os.path.exists(cmake_build_dir): + raise RuntimeError( + "CMake build directory does not exist: {}".format(cmake_build_dir) + ) + # Check that the CMake build directory has been configured and built + if not os.path.exists(os.path.join(cmake_build_dir, "CMakeCache.txt")): + raise RuntimeError( + "CMake build directory does not contain CMakeCache.txt: {}".format( + cmake_build_dir + ) + ) + # check for libnrniv.{so,dylib, dll.a} depending on platform + check_suffix = "so" if sys.platform != "darwin" else "dylib" + if sys.platform == "win32": + check_suffix = "dll.a" + if not os.path.exists( + os.path.join( + cmake_build_dir, + "lib", + "libnrniv.{}".format(check_suffix), + ) + ): + raise RuntimeError( + "CMake build directory does not contain libnrniv: {}".format( + cmake_build_dir + ) + ) + just_extensions = True + +# Check for RX3D Optimization Level +rx3d_opt_level = "-O0" +if "--rx3d-opt-level" in sys.argv: + rx3d_opt_level_arg = sys.argv[sys.argv.index("--rx3d-opt-level") + 1] + rx3d_opt_level = "-O{}".format(int(rx3d_opt_level_arg)) + sys.argv.remove("--rx3d-opt-level") + sys.argv.remove(rx3d_opt_level_arg) + +# If NRN_ENABLE_PYTHON_DYNAMIC is ON, we will build the wheel without the nrnpython library +without_nrnpython = False +if "--without-nrnpython" in sys.argv: + without_nrnpython = True + sys.argv.remove("--without-nrnpython") # Main source of the version. Dont rename, used by Cmake try: @@ -57,6 +108,7 @@ class Components: if "--disable-mpi" in sys.argv: Components.MPI = False + Components.MUSIC = False sys.argv.remove("--disable-mpi") if "--enable-coreneuron" in sys.argv: @@ -67,6 +119,10 @@ class Components: Components.GPU = True sys.argv.remove("--enable-gpu") +if "--enable-music" in sys.argv: + Components.MUSIC = True + sys.argv.remove("--enable-music") + if Components.RX3D: try: from Cython.Distutils import Extension as CyExtension @@ -112,7 +168,7 @@ def __init__( Extension.__init__(self, name, sources, **kw) self.sourcedir = os.path.abspath(cmake_proj_dir) self.cmake_flags = cmake_flags or [] - self.cmake_install_prefix = os.path.abspath("build/cmake_install") + self.cmake_install_prefix = os.path.abspath(cmake_build_dir) self.cmake_collect_dirs = cmake_collect_dirs or [] self.cmake_install_python_files = cmake_install_python_files self.cmake_done = False @@ -143,9 +199,18 @@ def run(self, *args, **kw): if isinstance(ext, CMakeAugmentedExtension): if ext.cmake_done: continue - - self._run_cmake(ext) # Build cmake project - + if just_extensions: + self.build_temp = cmake_build_dir + self.build_base = cmake_build_dir + # Make the temp include paths in the building the extension + ext.include_dirs += [ + os.path.join(self.build_temp, inc_dir) + for inc_dir in ext.include_dirs + if not os.path.isabs(inc_dir) + ] + continue + else: + self._run_cmake(ext) # Build cmake project if self.docs: return @@ -153,6 +218,7 @@ def run(self, *args, **kw): # These go directly into final package, regardless of setuptools filters logging.info("\n==> Collecting CMAKE files") rel_package = ext.name.split(".")[:-1] + package_data_d = os.path.join( self.build_lib, *(rel_package + [".data"]) ) @@ -214,6 +280,7 @@ def _run_cmake(self, ext): env["CXXFLAGS"] = "{} -DVERSION_INFO='{}'".format( env.get("CXXFLAGS", ""), self.distribution.get_version() ) + if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) @@ -230,14 +297,20 @@ def _run_cmake(self, ext): # RTD will call sphinx for us. We just need notebooks and doxygen if os.environ.get("READTHEDOCS"): subprocess.check_call( - ["make", "notebooks"], cwd=self.build_temp, env=env + [cmake, "--build", ".", "--target", "notebooks"], + cwd=self.build_temp, + env=env, ) subprocess.check_call( - ["make", "doxygen"], cwd=self.build_temp, env=env + [cmake, "--build", ".", "--target", "doxygen"], + cwd=self.build_temp, + env=env, ) else: subprocess.check_call( - ["make", "docs"], cwd=self.build_temp, env=env + [cmake, "--build", ".", "--target", "docs"], + cwd=self.build_temp, + env=env, ) else: subprocess.check_call( @@ -286,18 +359,11 @@ def _run_cmake(self, ext): @staticmethod def _find_cmake(): - for candidate in ["cmake", "cmake3"]: - try: - out = subprocess.check_output([candidate, "--version"]) - cmake_version = Version( - re.search(r"version\s*([\d.]+)", out.decode()).group(1) - ) - if cmake_version >= Version("3.15.0"): - return candidate - except OSError: - pass + cmake_cmd = which("cmake") or which("cmake3") + if not cmake_cmd: + raise RuntimeError("Project requires CMake") - raise RuntimeError("Project requires CMake >=3.15.0") + return cmake_cmd class Docs(Command): @@ -341,12 +407,28 @@ def setup_package(): REL_RPATH = "@loader_path" if sys.platform[:6] == "darwin" else "$ORIGIN" + ext_common_libraries = ["nrniv"] + if not without_nrnpython: + nrn_python_lib = "nrnpython{}".format( + sys.version_info[0] + if sys.platform != "win32" + else str(sys.version_info[0]) + str(sys.version_info[1]) + ) + ext_common_libraries.append(nrn_python_lib) + extension_common_params = defaultdict( list, - library_dirs=["build/cmake_install/lib"], - libraries=["nrniv", "nrnpython{}".format(sys.version_info[0])], + library_dirs=[os.path.join(cmake_build_dir, "lib")], + libraries=ext_common_libraries, + language="c++", ) + logging.info("Extension common compile flags %s" % str(extension_common_params)) + + # Get extra_compile_args and extra_link_args from environment variable + extra_link_args = os.environ.get("LDFLAGS", "").split() + extra_compile_args = os.environ.get("CFLAGS", "").split() + extensions = [ CMakeAugmentedExtension( "neuron.hoc", @@ -382,30 +464,42 @@ def setup_package(): "src/nrnpython", "src/nrnmpi", ], - extra_link_args=[ - # use relative rpath to .data/lib - "-Wl,-rpath,{}".format(REL_RPATH + "/.data/lib/") + extra_compile_args=extra_compile_args + ["-std=c++17"], + extra_link_args=extra_link_args + + [ + "-Wl,-rpath,{}{}".format( + REL_RPATH, "/../../" if just_extensions else "/.data/lib/" + ) ], **extension_common_params, ) ] + if Components.MUSIC: + extensions += [ + CyExtension( + "neuronmusic", + ["src/neuronmusic/neuronmusic.pyx"], + include_dirs=["src/nrnpython", "src/nrnmusic"], + **extension_common_params, + ) + ] + if Components.RX3D: include_dirs = ["share/lib/python/neuron/rxd/geometry3d", numpy.get_include()] # Cython files take a long time to compile with O2 so default O0 # But pay the price if uploading distribution - extra_compile_args = ["-O2" if "NRN_BUILD_FOR_UPLOAD" in os.environ else "-O0"] rxd_params = extension_common_params.copy() rxd_params["libraries"].append("rxdmath") rxd_params.update( dict( # Cython files take a long time to compile with O2 but this # is a distribution... - extra_compile_args=extra_compile_args, - extra_link_args=[ - "-Wl,-rpath,{}".format(REL_RPATH + "/../../.data/lib/") - ], + extra_compile_args=extra_compile_args + + ["-O2" if "NRN_BUILD_FOR_UPLOAD" in os.environ else rx3d_opt_level], + extra_link_args=extra_link_args + + ["-Wl,-rpath,{}".format(REL_RPATH + "/../../.data/lib/")], ) ) @@ -460,7 +554,7 @@ def setup_package(): if f[0] != "_" ], cmdclass=dict(build_ext=CMakeAugmentedBuilder, docs=Docs), - install_requires=["numpy>=1.9.3"] + maybe_patchelf, + install_requires=["numpy>=1.9.3", "packaging"] + maybe_patchelf, tests_require=["flake8", "pytest"], setup_requires=["wheel"] + maybe_docs + maybe_test_runner + maybe_rxd_reqs, dependency_links=[], @@ -497,7 +591,7 @@ def mac_osx_setenv(): " Consider using an official Python build from python.org" ) if py_osx_framework is not None and explicit_target > py_osx_framework: - logging.warn( + logging.warning( "You are building wheels for macOS >={}; this is more " "restrictive than your Python framework, which supports " ">={}".format(fmt(explicit_target), fmt(py_osx_framework)) @@ -509,19 +603,19 @@ def mac_osx_setenv(): if py_osx_framework is None: py_osx_framework = (10, 15) if py_osx_framework < (10, 15): - logging.warn( + logging.warning( "C++17 support is required to build NEURON on macOS, " "therefore minimum MACOSX_DEPLOYMENT_TARGET version is 10.15." ) py_osx_framework = (10, 15) if py_osx_framework > (10, 15): - logging.warn( + logging.warning( "You are building a wheel with a Python built for macOS >={}. " "Your wheel won't run on older versions, consider using an " "official Python build from python.org".format(fmt(py_osx_framework)) ) macos_target = "%d.%d" % tuple(py_osx_framework[:2]) - logging.warn("Setting MACOSX_DEPLOYMENT_TARGET=%s", macos_target) + logging.warning("Setting MACOSX_DEPLOYMENT_TARGET=%s", macos_target) os.environ["MACOSX_DEPLOYMENT_TARGET"] = macos_target diff --git a/share/lib/python/neuron/rxd/geometry3d/CMakeLists.txt b/share/lib/python/neuron/rxd/geometry3d/CMakeLists.txt deleted file mode 100644 index 2a298da327..0000000000 --- a/share/lib/python/neuron/rxd/geometry3d/CMakeLists.txt +++ /dev/null @@ -1,135 +0,0 @@ -# ============================================================================= -# Cython (pyx) to CPP conversion -# ============================================================================= -# ~~~ -# The cpp files are consumed to make modules in src/nrnpython/setup.py -# -# Because of problems building modules with MINGW containing Cython -# generated files, the cpp files are consumed to make modules from here, -# so that these modules can be built with the msvc toolchain. -# Maybe someday the src/nrnpython/setup.py -# can build all the extension modules for windows. -# -# Following set in src/nrnpython/CMakeLists.txt should be made global. -# These are used in setup.py. -# ~~~ -set(NRN_SRCDIR ${PROJECT_SOURCE_DIR}) -set(CC ${CMAKE_C_COMPILER}) -set(CXX ${CMAKE_CXX_COMPILER}) - -if(NRN_WINDOWS_BUILD) - set(BUILD_MINGW_TRUE "") - set(BUILD_MINGW_FALSE "#") -else() - set(BUILD_MINGW_TRUE "#") - set(BUILD_MINGW_FALSE "") -endif() - -# generate setup.py from setup.py.in -nrn_configure_file(setup.py share/lib/python/neuron/rxd/geometry3d) - -set(basenames ctng surfaces graphicsPrimitives) - -if(NRN_ENABLE_RX3D - AND NRN_ENABLE_MODULE_INSTALL - AND NRN_ENABLE_PYTHON) - # ~~~ - # For MINGW, python3 uses msvc - # The following shell script carries out the craziness when - # ${pyexe} setup.py build_ext --build-lib=${NRN_PYTHON_BUILD_LIB} - # may not be enough - # ~~~ - - # Prepare a shell script to transform cython generated cpp files for mingw - file( - WRITE ${CMAKE_CURRENT_BINARY_DIR}/cy_cpp_filt.sh - "\ -#!bash\n\ -set -ex\n\ -file=$1\n\ -mingw=${MINGW}\n\ -if test x$mingw = x1 ; then #only MINGW, linux/mac does nothing \n\ - if ! grep -q '_hypot' $file ; then #only transform once\n\ - echo '#define _hypot hypot' > $file.tmp\n\ - cat $file >> $file.tmp\n\ - sed 's/EXTERN_C DL_IMPORT(\\([^)]*\\))/EXTERN_C \\1/' $file.tmp > $file\n\ - rm $file.tmp - fi\n\ -fi\n\ -") - - # Prepare a shell script to run python setup.py - file( - WRITE ${CMAKE_CURRENT_BINARY_DIR}/runpy.sh - "\ -#!bash\n\ -set -ex\n\ -echo runpy $*\n\ -pyexe=$1\n\ -mingw=${MINGW}\n\ -shift\n\ -export LDCSHARED=\"${CMAKE_C_COMPILER} ${CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS}\"\n\ -export LDCXXSHARED=\"${CMAKE_CXX_COMPILER} ${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS}\"\n\ -if test x$mingw = x1 ; then\n\ - pyver=`$pyexe -c 'import sys; print (sys.version_info[0]); quit()'`\n\ - echo pyver=$pyver\n\ - if test x$pyver = x3 ; then # python3.x builds with msvc\n\ - . ${PROJECT_SOURCE_DIR}/mingw_files/vcenv.sh\n\ - $pyexe setup.py build_ext --build-lib=${NRN_PYTHON_BUILD_LIB}\n\ - fi\n\ -else #mac/linux does not need anything special\n\ - $pyexe setup.py build_ext --build-lib=${NRN_PYTHON_BUILD_LIB}\n\ -fi\n\ -") - - # ~~~ - # Cython generates *.cpp from *.pyx. Make a list of the cpp files - # so the rxd_extensions targets can depend on them. - # These cpp files are given a false dependency on setup.py so that a change - # to setup.py.in in the end causes setup.py to execute and which apparently - # rebuilds only if a cpp file has changed. - # ~~~ - foreach(basename ${basenames}) - set(bname ${CMAKE_CURRENT_BINARY_DIR}/${basename}.cpp) - set(sname ${CMAKE_CURRENT_SOURCE_DIR}/${basename}.pyx) - - add_custom_command( - OUTPUT ${bname} - COMMAND ${CYTHON_EXECUTABLE} -2 ${basename}.pyx -o ${bname} - COMMAND bash ${CMAKE_CURRENT_BINARY_DIR}/cy_cpp_filt.sh "${bname}" - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - DEPENDS ${sname} ${CMAKE_CURRENT_BINARY_DIR}/setup.py) - - list(APPEND rxd_cython_cpp_files ${bname}) - - endforeach(basename) - - # Want the cython custom command to generate only once no matter how many pythons - add_custom_target(rxd_cython_generated DEPENDS ${rxd_cython_cpp_files}) - - # ~~~ - # For each python detected / provided by user, build the extensions - # we do not care about the target names but they must be unique, hence the - # index at the end of each name. Notice that the unique target runs - # its COMMAND only if a DEPENDS is out of date (which is the case if setup.py.in) - # is out of date (see the CYTHON executable custom_command) - # ~~~ - list(LENGTH NRN_PYTHON_EXE_LIST _num_pythons) - math(EXPR num_pythons "${_num_pythons} - 1") - foreach(val RANGE ${num_pythons}) - list(GET NRN_PYTHON_EXE_LIST ${val} pyexe) - add_custom_target( - rx3dextensions_${val} - COMMAND bash ${CMAKE_CURRENT_BINARY_DIR}/runpy.sh "${pyexe}" - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS rxd_cython_generated) - list(APPEND rx3dextensions rx3dextensions_${val}) - endforeach(val) - - # ~~~ - # Always out of date but that does not imply any of the rx3dextension targets - # are out of date - # ~~~ - add_custom_target(rx3d ALL DEPENDS ${rx3dextensions}) - -endif() diff --git a/share/lib/python/neuron/rxd/geometry3d/setup.py.in b/share/lib/python/neuron/rxd/geometry3d/setup.py.in deleted file mode 100644 index 82541c76f0..0000000000 --- a/share/lib/python/neuron/rxd/geometry3d/setup.py.in +++ /dev/null @@ -1,92 +0,0 @@ -# run with -# python setup.py build_ext --inplace - -import sys -nrn_srcdir = "@NRN_SRCDIR@" -@USING_PGI_COMPILER_TRUE@using_pgi=True -@USING_PGI_COMPILER_FALSE@using_pgi=False -@BUILD_MINGW_TRUE@mingw = 1 -@BUILD_MINGW_FALSE@mingw = 0 -instdir = "@prefix@" -if nrn_srcdir[0] == '/' or (len(nrn_srcdir) > 2 and nrn_srcdir[1] == ':'): - pass -else: # not an absolute path - # TODO: is this right? - nrn_srcdir = '../../../../../../' + nrn_srcdir - -pgi_compiler_flags = "-noswitcherror" - -from setuptools import setup, Extension - -def have_vc(): - if not mingw: - return False - import traceback - try: - from shutil import which - x = which("cl") - x = True if x is not None and "Microsoft" in x else False - except: - traceback.print_exc() - x = False - return x - -try: - import numpy -except: - setup() -else: - olevel = "@NRN_RX3D_OPT_LEVEL@" - use_vc = have_vc() - if mingw and sys.version_info[0] == 3: - use_vc = True - if use_vc: - mpicc_bin = 'cl' - mpicxx_bin = 'cl' - if olevel == '0' and sys.version_info[0] == 3: - olevel = 'd' - else: - mpicc_bin = "@CC@" - mpicxx_bin = "@CXX@" - print (mpicxx_bin) - import os - os.environ["CC"]=mpicc_bin - os.environ["CXX"]=mpicxx_bin - - define_macros=[] - - extra_compile_args = [@NRN_COMPILE_FLAGS_COMMA_SEPARATED_STRINGS@] - extra_link_args = [@NRN_LINK_FLAGS_COMMA_SEPARATED_STRINGS@] - if olevel != "": - extra_compile_args.append('-O' + olevel) - if using_pgi: - extra_compile_args.append(pgi_compiler_flags) - - include_dirs = [nrn_srcdir + '/share/lib/python/neuron/rxd/geometry3d', '.', numpy.get_include()] - - srcdir = nrn_srcdir + '/share/lib/python/neuron/rxd/geometry3d/' - - # name='neuron/rxd/geometry3d', - setup( - ext_modules = [ - Extension("neuron.rxd.geometry3d.graphicsPrimitives", - sources=["graphicsPrimitives.cpp"], - define_macros=define_macros, - extra_compile_args=extra_compile_args, - extra_link_args=extra_link_args, - include_dirs=include_dirs), - Extension("neuron.rxd.geometry3d.ctng", - sources=["ctng.cpp"], - define_macros=define_macros, - extra_compile_args=extra_compile_args, - extra_link_args=extra_link_args, - include_dirs=include_dirs), - Extension("neuron.rxd.geometry3d.surfaces", - sources=["surfaces.cpp", nrn_srcdir + "/src/nrnpython/rxd_marching_cubes.cpp", nrn_srcdir + "/src/nrnpython/rxd_llgramarea.cpp"], - define_macros=define_macros, - extra_compile_args=extra_compile_args, - extra_link_args=extra_link_args, - include_dirs=include_dirs)], - ) - # package_dir = {'': instdir + '/share/lib/python/neuron/rxd/geometry3d'} - diff --git a/src/neuronmusic/CMakeLists.txt b/src/neuronmusic/CMakeLists.txt deleted file mode 100644 index e83c8d1d5e..0000000000 --- a/src/neuronmusic/CMakeLists.txt +++ /dev/null @@ -1,87 +0,0 @@ -# a lot of this copied from nrn/share/lib/python/neuron/rxd/geometry3d/CMakeLists.txt - -# ~~~ -# Following set in src/nrnpython/CMakeLists.txt should be made global. -# These are used in setup.py. -# ~~~ -set(NRN_SRCDIR ${PROJECT_SOURCE_DIR}) -set(CC ${CMAKE_C_COMPILER}) -set(CXX ${CMAKE_CXX_COMPILER}) - -nrn_configure_file(setup.py src/neuronmusic) - -if(NRN_ENABLE_MUSIC - AND NRN_ENABLE_MODULE_INSTALL - AND NRN_ENABLE_PYTHON) - - # Prepare a shell script to run python setup.py - file( - WRITE ${CMAKE_CURRENT_BINARY_DIR}/runpy.sh - "\ -#!bash\n\ -set -ex\n\ -echo runpy $*\n\ -pyexe=$1\n\ -mingw=${MINGW}\n\ -shift\n\ -export LDCSHARED=\"${CMAKE_C_COMPILER} ${CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS}\"\n\ -export LDCXXSHARED=\"${CMAKE_CXX_COMPILER} ${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS}\"\n\ -if test x$mingw = x1 ; then\n\ - pyver=`$pyexe -c 'import sys; print (sys.version_info[0]); quit()'`\n\ - echo pyver=$pyver\n\ - if test x$pyver = x3 ; then # python3.x builds with msvc\n\ - . ${PROJECT_SOURCE_DIR}/mingw_files/vcenv.sh\n\ - $pyexe setup.py build_ext --build-lib=${NRN_PYTHON_BUILD_LIB}\n\ - fi\n\ -else #mac/linux does not need anything special\n\ - $pyexe setup.py build --build-lib=${NRN_PYTHON_BUILD_LIB}\n\ -fi\n\ -") - - # ~~~ - # Cython generates *.cpp from *.pyx. Make a list of the cpp files - # so the rxd_extensions targets can depend on them. - # These cpp files are given a false dependency on setup.py so that a change - # to setup.py.in in the end causes setup.py to execute and which apparently - # rebuilds only if a cpp file has changed. - # ~~~ - - set(name neuronmusic) - set(cppname ${CMAKE_CURRENT_BINARY_DIR}/neuronmusic.cpp) - set(pyxname ${CMAKE_CURRENT_SOURCE_DIR}/neuronmusic.pyx) - - add_custom_command( - OUTPUT ${cppname} - COMMAND ${CYTHON_EXECUTABLE} -2 ${name}.pyx -o ${cppname} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - DEPENDS ${pyxname} ${CMAKE_CURRENT_BINARY_DIR}/setup.py) - - # Want the cython custom command to generate only once no matter how many pythons - add_custom_target(neuronmusic_cython_generated DEPENDS ${cppname}) - - add_custom_target( - neuronmusicextension ALL - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS ${cppname}) - - # ~~~ - # For each python detected / provided by user, build the extensions - # we do not care about the target names but they must be unique, hence the - # index at the end of each name. Notice that the unique target runs - # its COMMAND only if a DEPENDS is out of date (which is the case if setup.py.in) - # is out of date (see the CYTHON executable custom_command) - # ~~~ - foreach(pyexe ${NRN_PYTHON_EXE_LIST}) - add_custom_command( - TARGET neuronmusicextension - POST_BUILD - COMMAND bash ${CMAKE_CURRENT_BINARY_DIR}/runpy.sh "${pyexe}" - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "Building neuronmusic module with: ${pyexe}") - endforeach(pyexe) - - add_dependencies(neuronmusicextension nrniv_lib neuronmusic_cython_generated) - # install modules from NRN_PYTHON_BUILD_LIB - install(DIRECTORY ${NRN_PYTHON_BUILD_LIB} DESTINATION lib) - -endif() diff --git a/src/nrnpython/CMakeLists.txt b/src/nrnpython/CMakeLists.txt index 8863183580..082fa0ab75 100644 --- a/src/nrnpython/CMakeLists.txt +++ b/src/nrnpython/CMakeLists.txt @@ -5,52 +5,8 @@ add_compile_definitions(${NRN_COMPILE_DEFS}) add_link_options(${NRN_LINK_FLAGS}) # ============================================================================= -# Definitions used in setup.py +# List of python executables to build nrnpython against # ============================================================================= -set(NRN_SRCDIR ${PROJECT_SOURCE_DIR}) -set(NRNPYTHON_DEFINES "") - -# ~~~ -# If NRN_PYTHON_DYNAMIC then the following three will be determined from the -# actual pyexe that runs setup.py -# ~~~ -set(NRNPYTHON_EXEC ${PYTHON_EXECUTABLE}) -set(NRNPYTHON_PYVER ${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}) -set(npy_pyver10 "") - -# reset since no " allowed -set(PACKAGE_VERSION ${PROJECT_VERSION}) -set(NRN_LIBDIR ${CMAKE_INSTALL_PREFIX}/lib) - -# set by IV's cmake module -set(IV_LIBDIR "${IV_INCLUDE_DIR}/../lib") - -# no suffix for readline -set(READLINE_SOSUFFIX "") - -if(NRN_ENABLE_RX3D) - set(BUILD_RX3D "1") -else() - set(BUILD_RX3D "0") -endif() - -if(NRN_ENABLE_INTERVIEWS) - set(IVHINES "interviews") -else() - set(IVHINES "") -endif() - -# use compiler from CMake -set(CC ${CMAKE_C_COMPILER}) -set(CXX ${CMAKE_CXX_COMPILER}) - -# ============================================================================= -# Options that get expanded to comments -# ============================================================================= -set(BUILD_NRNMPI_DYNAMIC_FALSE "#") -set(BUILD_NRNPYTHON_DYNAMIC_TRUE "") -set(BUILD_NRNPYTHON_DYNAMIC_FALSE "#") - if(NRN_ENABLE_PYTHON_DYNAMIC) if("${NRN_PYTHON_DYNAMIC}" STREQUAL "") set(NRN_PYTHON_EXE_LIST @@ -62,30 +18,14 @@ if(NRN_ENABLE_PYTHON_DYNAMIC) CACHE INTERNAL "" FORCE) endif() else() - set(BUILD_NRNPYTHON_DYNAMIC_TRUE "#") - set(BUILD_NRNPYTHON_DYNAMIC_FALSE "") set(NRN_PYTHON_EXE_LIST ${PYTHON_EXECUTABLE} CACHE INTERNAL "" FORCE) endif() -if(NRN_MACOS_BUILD) - set(MAC_DARWIN_TRUE "") - set(MAC_DARWIN_FALSE "#") -else() - set(MAC_DARWIN_TRUE "#") - set(MAC_DARWIN_FALSE "") -endif() - -if(NRN_WINDOWS_BUILD) - set(BUILD_MINGW_TRUE "") - set(BUILD_MINGW_FALSE "#") -else() - set(BUILD_MINGW_TRUE "#") - set(BUILD_MINGW_FALSE "") -endif() - +# ============================================================================= # rxdmath libraries (always build) +# ============================================================================= add_library(rxdmath SHARED ${CMAKE_CURRENT_SOURCE_DIR}/rxdmath.cpp) install(TARGETS rxdmath DESTINATION ${NRN_INSTALL_SHARE_LIB_DIR}) @@ -142,7 +82,7 @@ endif() # Install package files that were created in build (e.g. .py.in) install( DIRECTORY ${PROJECT_BINARY_DIR}/share/lib/python - DESTINATION ${NRN_LIBDIR} + DESTINATION lib FILES_MATCHING PATTERN *.py PATTERN *.so @@ -154,9 +94,6 @@ install( # ============================================================================= if(NRN_ENABLE_MODULE_INSTALL) - # All variables are set, prepare setup.py for python module install - nrn_configure_file(setup.py src/nrnpython) - # Setup MinGW toolchain for setuptools if(MINGW) file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/setup.cfg "[build]\ncompiler=mingw32") @@ -178,6 +115,7 @@ if(NRN_ENABLE_MODULE_INSTALL) # need to copy inithoc.cpp in from source to binary dir if any module # dependent changes and make a custom target as well. # ~~~ + set(binary_dir_filename ${CMAKE_CURRENT_BINARY_DIR}/inithoc.cpp) set(source_dir_filename ${CMAKE_CURRENT_SOURCE_DIR}/inithoc.cpp) set(inithoc_hdeps @@ -188,7 +126,7 @@ if(NRN_ENABLE_MODULE_INSTALL) OUTPUT ${binary_dir_filename} COMMAND cp ${source_dir_filename} ${binary_dir_filename} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - DEPENDS ${source_dir_filename} ${CMAKE_CURRENT_BINARY_DIR}/setup.py ${inithoc_hdeps}) + DEPENDS ${source_dir_filename} ${inithoc_hdeps}) add_custom_target( hoc_module ALL @@ -220,33 +158,75 @@ if(NRN_ENABLE_MODULE_INSTALL) # ============================================================================= # for each python detected / provided by user, install module at install time - if(NRN_SANITIZERS) - # Make sure the same compiler (currently always clang in the sanitizer builds) is used to link - # as was used to build. By default, Python will try to use the compiler that was used to build - # Python, which is often gcc. - list(APPEND extra_env "LDCSHARED=${CMAKE_C_COMPILER} ${CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS}" - "LDCXXSHARED=${CMAKE_CXX_COMPILER} ${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS}") + # setup.py tries to compile C++ code with the C compiler. This works with gcc and intel but fails + # with at least PGI/NVHPC compiler. Hence, use C++ compiler if PGI/NVHPC compiler is used. + if(NRN_HAVE_NVHPC_COMPILER) + list(APPEND extra_env "CC=${CMAKE_CXX_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}") + else() + list(APPEND extra_env "CC=${CMAKE_C_COMPILER}" "CXX=${CMAKE_CXX_COMPILER}") + endif() + + if(NRN_COMPILE_FLAGS) + list(APPEND extra_env "CFLAGS=${NRN_COMPILE_FLAGS_STRING}") + endif() + if(NRN_LINK_FLAGS) + list(APPEND extra_env "LDFLAGS=${NRN_LINK_FLAGS_STRING}") + endif() + # Make sure the same compiler (currently always clang in the sanitizer builds) is used to link as + # was used to build. By default, Python will try to use the compiler that was used to build + # Python, which is often gcc. + list(APPEND extra_env "LDCSHARED=${CMAKE_C_COMPILER} ${CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS}" + "LDCXXSHARED=${CMAKE_CXX_COMPILER} ${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS}") + + # Prepare build options for setup.py build + set(NRN_SETUP_PY_BUILD_OPTIONS "--cmake-build-dir") + list(APPEND NRN_SETUP_PY_BUILD_OPTIONS ${CMAKE_BINARY_DIR}) + if(NOT NRN_ENABLE_RX3D) + list(APPEND NRN_SETUP_PY_BUILD_OPTIONS "--disable-rx3d") + else() + list(APPEND NRN_SETUP_PY_BUILD_OPTIONS "--rx3d-opt-level" ${NRN_RX3D_OPT_LEVEL}) + endif() + if(MINGW) + list(APPEND NRN_SETUP_PY_BUILD_OPTIONS "--compiler=mingw32") + endif() + if(NOT NRN_ENABLE_PYTHON_DYNAMIC) + list(APPEND NRN_SETUP_PY_BUILD_OPTIONS "--without-nrnpython") + endif() + if(NRN_ENABLE_MUSIC) + list(APPEND NRN_SETUP_PY_BUILD_OPTIONS "--enable-music") + endif() + list(APPEND NRN_SETUP_PY_BUILD_OPTIONS "--build-lib=${NRN_PYTHON_BUILD_LIB}") + + # Prepare build_ext options for setup.py build + set(NRN_SETUP_PY_BUILD_EXT_OPTIONS "build_ext") + if(NRN_ENABLE_MUSIC) + # NRN_SETUP_PY_INC_DIRS is crafted for the MUSIC build, to be extended to other if needed + set(NRN_SETUP_PY_INC_DIRS "${MUSIC_INCDIR};${MPI_INCLUDE_PATH}") + string(REPLACE ";" ":" NRN_SETUP_PY_INC_DIRS "${NRN_SETUP_PY_INC_DIRS}") + list(APPEND NRN_SETUP_PY_BUILD_EXT_OPTIONS "--include-dirs=${NRN_SETUP_PY_INC_DIRS}") + endif() + if(MINGW) + list(APPEND NRN_SETUP_PY_BUILD_EXT_OPTIONS "--define=MS_WIN64") + endif() + # force rebuild of cython generated files for PYTHON_DYNAMIC + if(NRN_ENABLE_PYTHON_DYNAMIC) + list(APPEND NRN_SETUP_PY_BUILD_EXT_OPTIONS "--force") endif() + # Build python module foreach(pyexe ${NRN_PYTHON_EXE_LIST}) add_custom_command( TARGET hoc_module POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/inithoc.cpp ${CMAKE_CURRENT_BINARY_DIR}/inithoc.cpp - COMMAND - ${CMAKE_COMMAND} -E copy_if_different - ${PROJECT_SOURCE_DIR}/share/lib/python/neuron/help_data.dat - ${CMAKE_CURRENT_BINARY_DIR}/lib/python/neuron/help_data.dat - COMMAND ${extra_env} ${pyexe} setup.py --quiet build --build-lib=${NRN_PYTHON_BUILD_LIB} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "Building python module with: ${pyexe}") + COMMAND ${CMAKE_COMMAND} -E env ${extra_env} ${pyexe} setup.py build + ${NRN_SETUP_PY_BUILD_OPTIONS} ${NRN_SETUP_PY_BUILD_EXT_OPTIONS} + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + COMMENT "Building python module with: ${pyexe} setup.py build + ${NRN_SETUP_PY_BUILD_OPTIONS} ${NRN_SETUP_PY_BUILD_EXT_OPTIONS}") endforeach(pyexe) - add_dependencies(hoc_module nrniv_lib ${nrnpython_lib_list}) - if(NRN_ENABLE_RX3D) - add_dependencies(hoc_module rxd_cython_generated) - endif() - # install modules from NRN_PYTHON_BUILD_LIB + add_dependencies(hoc_module nrniv_lib rxdmath ${nrnpython_lib_list}) install(DIRECTORY ${NRN_PYTHON_BUILD_LIB} DESTINATION lib) endif() diff --git a/src/nrnpython/setup.py.in b/src/nrnpython/setup.py.in deleted file mode 100644 index 4c38189c0c..0000000000 --- a/src/nrnpython/setup.py.in +++ /dev/null @@ -1,202 +0,0 @@ -#setup.py -from setuptools import setup, Extension -from sysconfig import get_python_version - -import sys -import os - -@USING_PGI_COMPILER_TRUE@using_pgi=True -@USING_PGI_COMPILER_FALSE@using_pgi=False -@BUILD_MINGW_TRUE@mingw = 1 -@BUILD_MINGW_FALSE@mingw = 0 - -# TODO temporary workaround for mingw + cmake -# When running with mingw-cmake, the path of the form C:/msys64/xyz -# needs to be transformed to C:\msys64/xyz. -def get_escaped_path(path): - if mingw: - return path.replace(":/", ":\\") - else: - return path - -# NRNPYTHON_DEFINES which were enabled at configure time -extern_defines = "@NRNPYTHON_DEFINES@" -nrnpython_exec = get_escaped_path("@NRNPYTHON_EXEC@") -nrnpython_pyver = "@NRNPYTHON_PYVER@" -nrnpython_pyver10 = "nrnpython" -use_libnrnpython_majorminor = @USE_LIBNRNPYTHON_MAJORMINOR@ -nrn_srcdir = get_escaped_path("@NRN_SRCDIR@") -build_rx3d = @BUILD_RX3D@ -ivlibdir = get_escaped_path("@IV_LIBDIR@") -if ivlibdir == "" : - ivlibdir = '.' - -destdir = os.getenv("DESTDIR") -if not destdir: - destdir = "" - -# install into build directory and then copy to install prefix -instdir = destdir + get_escaped_path("@CMAKE_BINARY_DIR@") - -if nrn_srcdir[0] != '/' : - if mingw: - nrn_srcdir = nrn_srcdir - else: - nrn_srcdir = '../../' + nrn_srcdir - -if "@BUILD_NRNPYTHON_DYNAMIC_TRUE@" == "": - # can do all the setup.py without re-configure. - nrnpython_pyver = get_python_version() - nrnpython_exec = sys.executable - nrnpython_pyver10 = "nrnpython" + str(sys.version_info[0]) - if use_libnrnpython_majorminor == 1: - nrnpython_pyver10 = "nrnpython" + str(sys.version_info[0]) + str(sys.version_info[1]) - -else: - if nrnpython_pyver!=get_python_version(): - print ("Error:") - print ("NEURON configure time python: "+nrnpython_exec+" "+ nrnpython_pyver) - print ("Python presently executing setup.py: "+sys.executable+" "+ get_python_version()) - print ("These do not match, and they should!") - sys.exit(1) - - -ldefs = extern_defines.split('-D') - -# if using MPI then at least for linking need special paths and libraries -import os -c_compiler_path = get_escaped_path("@CC@") -cxx_compiler_path = get_escaped_path("@CXX@") - -# setup.py tries to compile C++ code with the C compiler. This works with -# gcc and intel but fails with at least PGI compiler. Hence, use C++ compiler -# if PGI compiler is used. -os.environ['CC'] = cxx_compiler_path if using_pgi else c_compiler_path -os.environ["CXX"] = cxx_compiler_path - -if using_pgi: - # Spack installed Python includes patches that require setting LDSHARED - # and LDCXXSHARED environmental variables to ensure the PGI/NVIDIA linker - # is used as well as the compiler. Otherwise GCC is used and it chokes on - # PGI/NVIDIA-specific compiler flags. - os.environ["LDSHARED"] = os.environ['CC'] + " -shared" - os.environ["LDCXXSHARED"] = os.environ["CXX"] + " -shared" - - -#include_dirs for hoc module -include_dirs = [nrn_srcdir+'/src/nrnpython', nrn_srcdir+'/src/oc', '../oc', nrn_srcdir+'/src/nrnmpi'] -#include dirs for all modules - -include_dirs_common = [] - -#not needed with clang,clang++ and on my beta catalina does not -#work anyway with /Library/Developer/CommandLineTools/usr/bin/cc -#for the ctng module -#@MAC_DARWIN_TRUE@include_dirs_common.append("@CMAKE_OSX_SYSROOT@/usr/include") - -defines = [] - -libdirs = [destdir + get_escaped_path("@NRN_LIBDIR@"), - ivlibdir -] - -# prepare rpath flags for neuron and iv libs directories -rpath_prefix_flag='-Wl,-R' -extra_link_args = [@NRN_LINK_FLAGS_COMMA_SEPARATED_STRINGS@] -@MAC_DARWIN_FALSE@extra_link_args += [rpath_prefix_flag+lib_path for lib_path in libdirs] -@MAC_DARWIN_TRUE@extra_link_args.append("-Wl,-rpath,@loader_path/../../") -@MAC_DARWIN_TRUE@extra_link_args.append("-Wl,-rpath,%s" % ivlibdir) - -# as neuron module will be built during make, add build/lib -# directory for linking. Note that build/lib shouldn't be -# added to rpath to avoid issues with dlopen. -libdirs.append(destdir + get_escaped_path("@CMAKE_BINARY_DIR@/lib")) - -@MAC_DARWIN_FALSE@readline="readline@READLINE_SOSUFFIX@" -@MAC_DARWIN_TRUE@readline="readline@READLINE_SOSUFFIX@" - -pgi_compiler_flags = "-noswitcherror" - -libs = [@BUILD_NRNPYTHON_DYNAMIC_TRUE@nrnpython_pyver10, -"nrniv" -] -if "@IVHINES@" != "": - libs.append("@IVHINES@") - -extra_compile_args = [@NRN_COMPILE_FLAGS_COMMA_SEPARATED_STRINGS@] - -if using_pgi: - extra_link_args.append(pgi_compiler_flags) - extra_compile_args.append(pgi_compiler_flags) - -hoc_module = Extension( - "neuron.hoc", - ["inithoc.cpp"], - library_dirs=libdirs, - extra_link_args=extra_link_args, - #extra_objects = [], - extra_compile_args = extra_compile_args, - libraries = libs , - include_dirs = include_dirs+include_dirs_common, - define_macros=defines - ) - -ext_modules = [hoc_module] - -# The rx3d extensions are built using the setup.py.in in -# nrn/share/lib/python/neuron/rxd/geometry3d which contains Cython pyx files -# These files on Windows have to be compiled with the msvc compiler to avoid -# issues with hypot, a #define, and some DL_IMPORT statements. At least for -# Python3.5, the module must be built with the msvc toolchain to prevent a -# crash, possibly due to incompatible c runtime libraries. The following -# section is left here with the idea that someday the entire build on windows -# may be done with the msvc toolchain. Note, although the extensions are not -# built here, the python files are copied because of the call to setup here. - -if build_rx3d: - try: - import numpy - # TODO: do we need to use os.path.join? - src_path = nrn_srcdir + '/share/lib/python/neuron/rxd/geometry3d/' - build_path = '../../share/lib/python/neuron/rxd/geometry3d/' - include_dirs = [nrn_srcdir + '/share/lib/python/neuron/rxd/geometry3d', '.', numpy.get_include()] - include_dirs += include_dirs_common - extra_compile_args=["-O@NRN_RX3D_OPT_LEVEL@"] + extra_compile_args - define_macros = [] - if mingw: - #Avoid undefined __imp_Py_InitModule4 and hypot problem - define_macros.append(("MS_WIN64", None)) - ext_modules=[hoc_module, - Extension("neuron.rxd.geometry3d.graphicsPrimitives", - sources=[build_path + "graphicsPrimitives.cpp"], - extra_compile_args = extra_compile_args, - extra_link_args=extra_link_args, - define_macros = define_macros, - include_dirs=include_dirs), - Extension("neuron.rxd.geometry3d.ctng", - sources=[build_path + "ctng.cpp"], - extra_compile_args = extra_compile_args, - extra_link_args=extra_link_args, - define_macros = define_macros, - include_dirs=include_dirs), - Extension("neuron.rxd.geometry3d.surfaces", - sources=[build_path + "surfaces.cpp", nrn_srcdir + "/src/nrnpython/rxd_marching_cubes.cpp", nrn_srcdir + "/src/nrnpython/rxd_llgramarea.cpp"], - define_macros = define_macros, - extra_compile_args = extra_compile_args, - extra_link_args=extra_link_args, - include_dirs=include_dirs)] - except: - pass - -packages=['neuron','neuron.neuroml','neuron.tests', 'neuron.tests.utils', 'neuron.rxd', 'neuron.crxd', 'neuron.gui2'] -if build_rx3d: - packages +=['neuron.rxd.geometry3d'] - -setup(name="NEURON", version="@PACKAGE_VERSION@", - description = "NEURON bindings for python", - package_dir = {'':instdir+('/lib/python' if 'win32' in sys.platform else '/share/nrn/lib/python')}, - packages=packages, - data_files = [('neuron', [nrn_srcdir + '/share/lib/python/neuron/help_data.dat'])], - ext_modules=ext_modules -) - From 495c2f1d807a7e21c3b7691248da57aca7f6d8c6 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 10 Jan 2024 12:02:28 +0100 Subject: [PATCH 3/7] Fix wrong name --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e4bcfa0753..429b2ea6b4 100644 --- a/setup.py +++ b/setup.py @@ -585,7 +585,7 @@ def mac_osx_setenv(): if py_osx_framework is None: py_osx_framework = [10, 9] if py_osx_framework[1] > 9: - log.warn( + logging.warning( "[ WARNING ] You are building a wheel with a Python built" " for a recent MACOS version (from brew?). Your wheel won't be portable." " Consider using an official Python build from python.org" From 2a915574a8d799367e7f529d0ea440f49165d6ba Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 10 Jan 2024 12:09:04 +0100 Subject: [PATCH 4/7] Put back missing symbols --- setup.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/setup.py b/setup.py index 429b2ea6b4..dd137afe1b 100644 --- a/setup.py +++ b/setup.py @@ -580,6 +580,16 @@ def mac_osx_setenv(): logging.info("Setting SDKROOT=%s", sdk_root) os.environ["SDKROOT"] = sdk_root + def fmt(version): + return ".".join(str(x) for x in version) + + if "MACOSX_DEPLOYMENT_TARGET" in os.environ: + # Don't override the value if it is set explicitly, but try and print a + # helpful message + explicit_target = tuple( + int(x) for x in os.environ["MACOSX_DEPLOYMENT_TARGET"].split(".") + ) + # Match Python OSX framework py_osx_framework = extract_macosx_min_system_version(sys.executable) if py_osx_framework is None: From df3a8eb77e6a4d74df51bd0280886cbf4b02f4ea Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 11 Jan 2024 09:50:28 +0100 Subject: [PATCH 5/7] Fix Cython dep in `setup.py` --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index dd137afe1b..41c41a3e26 100644 --- a/setup.py +++ b/setup.py @@ -391,7 +391,7 @@ def setup_package(): NRN_COLLECT_DIRS = ["bin", "lib", "include", "share"] docs_require = [] # sphinx, themes, etc - maybe_rxd_reqs = ["numpy", "Cython<3"] if Components.RX3D else [] + maybe_rxd_reqs = ["numpy", "Cython"] if Components.RX3D else [] maybe_docs = docs_require if "docs" in sys.argv else [] maybe_test_runner = ["pytest-runner"] if "test" in sys.argv else [] From de9333d0f070c86f6d6a5f9db7c3a409f2134de5 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 11 Jan 2024 09:59:29 +0100 Subject: [PATCH 6/7] Fix version issues --- setup.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/setup.py b/setup.py index 41c41a3e26..648bccb89d 100644 --- a/setup.py +++ b/setup.py @@ -89,6 +89,18 @@ class Components: ) __version__ = v[: v.rfind("-")].replace("-", ".") if "-" in v else v + + # if version is not a valid PEP440 version, then create a bogus version + # that will be used only for development purposes which appends the commit hash + if not re.match(r"^\d+(\.\d+)*$", __version__): + __version__ = "0.0.dev0+g" + ( + subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], stdout=subprocess.PIPE + ) + .stdout.strip() + .decode() + ) + # allow to override version during development/testing if "NEURON_WHEEL_VERSION" in os.environ: __version__ = os.environ["NEURON_WHEEL_VERSION"] From 224410fb809ff0e855ba434c098da0e53d9538d8 Mon Sep 17 00:00:00 2001 From: Pramod Kumbhar Date: Thu, 11 Jan 2024 20:40:22 +0100 Subject: [PATCH 7/7] Reintroduce Cython<3 for ABI compatibility --- nrn_requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nrn_requirements.txt b/nrn_requirements.txt index cddce9b9dc..972947f029 100644 --- a/nrn_requirements.txt +++ b/nrn_requirements.txt @@ -5,7 +5,7 @@ matplotlib # bokeh 3 seems to break docs notebooks bokeh<3 ipython -cython +cython<3 packaging pytest pytest-cov diff --git a/setup.py b/setup.py index 648bccb89d..6b4f12476a 100644 --- a/setup.py +++ b/setup.py @@ -403,7 +403,7 @@ def setup_package(): NRN_COLLECT_DIRS = ["bin", "lib", "include", "share"] docs_require = [] # sphinx, themes, etc - maybe_rxd_reqs = ["numpy", "Cython"] if Components.RX3D else [] + maybe_rxd_reqs = ["numpy", "Cython<3"] if Components.RX3D else [] maybe_docs = docs_require if "docs" in sys.argv else [] maybe_test_runner = ["pytest-runner"] if "test" in sys.argv else []