From 953c4586c5ae0b74f3c93577320a29dea35b3d95 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Thu, 29 May 2025 12:53:32 -0400 Subject: [PATCH 1/9] fix install --- cFMS | 2 +- compile.py | 93 +++++++++++++++++++++++++++++++++++++++++++++ compile_c_libs.sh | 25 ------------ pyfms/cfms.py | 14 ++++--- run_tests.sh | 1 - setup.py | 20 ++-------- tests/test_build.py | 5 --- tests/test_init.py | 38 ++++-------------- 8 files changed, 113 insertions(+), 85 deletions(-) create mode 100644 compile.py delete mode 100755 compile_c_libs.sh delete mode 100644 tests/test_build.py diff --git a/cFMS b/cFMS index a344def..c0eb355 160000 --- a/cFMS +++ b/cFMS @@ -1 +1 @@ -Subproject commit a344def5420a3352da8bfbcbc0b0bc25b1e321db +Subproject commit c0eb3551f4200a3574460367fceed70d58de161c diff --git a/compile.py b/compile.py new file mode 100644 index 0000000..8e344fb --- /dev/null +++ b/compile.py @@ -0,0 +1,93 @@ +import os +import subprocess + +#path to the installed yaml library +yaml = "/opt/libyaml/0.2.5/GNU/14.2.0/" + +#path to the installed netcdf library +netcdf = "/opt/netcdf/4.9.3/GNU/14.2.0/" + +#Fortran compiler +FC = "mpif90" + +#C compiler +CC ="mpicc" + +#default Fortran compiler flags +FMS_FCFLAGS = f"-I{yaml}/include -I{netcdf}/include -fPIC" + +#default C compiler flags +FMS_CFLAGS = f"-I{yaml}/include -I{netcdf}/include -fPIC" + +#library flags +FMS_LDFLAGS = f"-L{yaml}/lib -L{netcdf}/lib -lnetcdf" + +cFMS_FCFLAGS = "-fPIC" +cFMS_CFLAGS = "-fPIC" +cFMS_LDFLAGS = "" + +#current directory +currdir = os.path.dirname(__file__) + +#absolute path to cFMS submodule +cFMS = f"{currdir}/cFMS" + +#absolute path to FMS submodule +FMS = f"{cFMS}/FMS" + +#absolue path to install libraries +cFMS_install = f"{currdir}/pyfms/lib/cFMS" +FMS_install = f"{currdir}/pyfms/lib/FMS" + + +def compile_FMS(): + + """ + Install FMS to FMS_install + """ + + currdir = os.path.dirname(__file__) + os.chdir(FMS) + + subprocess.run(["autoreconf", "-iv"]) + subprocess.run(["./configure", + "--enable-portable-kinds", + "--with-yaml", + f"FC={FC}", + f"CC={CC}", + f"FCFLAGS={FMS_FCFLAGS}", + f"CFLAGS={FMS_CFLAGS}", + f"LDFLAGS={FMS_LDFLAGS}", + f"--prefix={FMS_install}", + ]) + subprocess.run(["make", "install"]) + + os.chdir(currdir) + + +def compile_cFMS(): + + """ + Install cFMS to cFMS_install + """ + currdir = os.path.dirname(__file__) + os.chdir(cFMS) + + subprocess.run(["autoreconf", "-iv"]) + subprocess.run(["./configure", + f"--with-fms={FMS_install}", + f"FC={FC}", + f"CC={CC}", + f"FCFLAGS={cFMS_FCFLAGS}", + f"CFLAGS={cFMS_CFLAGS}", + F"LDFLAGS={cFMS_LDFLAGS}", + f"--prefix={cFMS_install}" + ]) + subprocess.run(["make", "install"]) + + os.chdir(currdir) + + +#compile +compile_FMS() +compile_cFMS() diff --git a/compile_c_libs.sh b/compile_c_libs.sh deleted file mode 100755 index 15eda09..0000000 --- a/compile_c_libs.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -curr_dir=$PWD/cFMS -install_fms=$curr_dir/FMS/LIBFMS - -export FC=mpif90 -export CC=mpicc - -cd $curr_dir/FMS -autoreconf -iv -export FCFLAGS="$FCFLAGS -fPIC" -export CFLAGS="$CFLAGS -fPIC" -./configure --enable-portable-kinds --with-yaml --prefix=$install_fms -make install - -cd .. - -export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$install_fms/lib" -export FCFLAGS="$FCFLAGS -I$install_fms/include" -export CFLAGS="$CFLAGS -I$install_fms/include" -export LDFLAGS="$LDFLAGS -lFMS -L$install_fms/lib" - -autoreconf -iv -./configure --prefix=$curr_dir/cLIBFMS -make install diff --git a/pyfms/cfms.py b/pyfms/cfms.py index 64e6ea8..b5de8b8 100644 --- a/pyfms/cfms.py +++ b/pyfms/cfms.py @@ -4,9 +4,6 @@ import pyfms -_libpath = os.path.dirname(__file__) + "/../cFMS/cLIBFMS/lib/libcFMS.so" -_lib = ctypes.cdll.LoadLibrary(_libpath) - def init(libpath: str = None): @@ -21,9 +18,16 @@ def init(libpath: str = None): global _libpath, _lib - if libpath is not None: + if libpath is None: + _libpath = os.path.dirname(__file__) + "/lib/cFMS/lib/libcFMS.so" + try: + _lib = ctypes.cdll.LoadLibrary(_libpath) + except: + print (f"_libpath does not exist. Please compile cFMS with ./compile.py\ + or provide a path to cFMS with pyfms.cfms.init(libpath=path_to_cfms_so") + else: _libpath = libpath - _lib = ctypes.cdll.LoadLibrary(_libpath) + _lib = ctypes.cdll.LoadLibrary(_libpath) pyfms.utils.constants._init(_libpath, _lib) pyfms.utils.grid_utils._init(_libpath, _lib) diff --git a/run_tests.sh b/run_tests.sh index 9ba3181..272b5d7 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -14,7 +14,6 @@ function run_test() { eval $1 if [ $? -ne 0 ] ; then exit 1 ; fi } -run_test "pytest tests/test_build.py" test="tests/test_fms.py" create_input $test diff --git a/setup.py b/setup.py index 40cedb8..1848dcf 100644 --- a/setup.py +++ b/setup.py @@ -1,19 +1,6 @@ -import subprocess -from typing import List - from setuptools import find_namespace_packages, setup -from setuptools.command.install import install - - -class CustomInstall(install): - def run(self): - # subprocess.run(["./compile_c_libs.sh"], capture_output=True, text=True) - with open("install.log", "w") as f: - subprocess.run(["./compile_c_libs.sh"], stdout=f) - install.run(self) - -test_requirements = ["pytest", "pytest-subtests", "coverage"] +test_requirements = ["pytest", "pytest-subtests", "coverage", "xarray", "netCDF4"] develop_requirements = test_requirements + ["pre-commit"] extras_requires = { @@ -21,7 +8,7 @@ def run(self): "develop": develop_requirements, } -requirements: List[str] = ["dacite", "h5netcdf", "numpy", "pyyaml", "mpi4py", "xarray"] +requirements = ["dacite", "numpy", "pyyaml", "mpi4py"] setup( author="NOAA/GFDL", @@ -38,9 +25,8 @@ def run(self): name="pyfms", license="", packages=find_namespace_packages(include=["pyfms", "pyfms.*"]), - cmdclass={"install": CustomInstall}, include_package_data=True, - url="https://github.com/fmalatino/pyFMS.git", + url="https://github.com/NOAA-GFDL/pyFMS.git", version="2024.12.0", zip_safe=False, ) diff --git a/tests/test_build.py b/tests/test_build.py deleted file mode 100644 index 7d6de4c..0000000 --- a/tests/test_build.py +++ /dev/null @@ -1,5 +0,0 @@ -import os - - -def test_shared_object_exists(): - assert os.path.exists(os.path.dirname(__file__) + "/../cFMS/cLIBFMS/lib/libcFMS.so") diff --git a/tests/test_init.py b/tests/test_init.py index a889aba..e86183e 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -5,17 +5,14 @@ import pyfms -def test_write_module(): - myfile = open("module1.py", "w") - myfile.write( - """ -import pyfms -class Module1Class(): - module1_lib_id = id(pyfms.cfms.lib()) - """ - ) - myfile.close() +def test_library_loaded(): + + """ + Test to ensure library loaded automatically + """ + assert pyfms.cfms._lib is not None + def test_share_same_library(): @@ -27,19 +24,6 @@ def test_share_same_library(): assert id(pyfms.cfms._lib) == id(pyfms.mpp_domains._lib) -def test_load_library_same_object(): - - """ - Test to ensure the ctypes CDLL Library object - is instantiated only once - """ - - import module1 - - myclass = module1.Module1Class() - assert id(pyfms.cfms.lib()) == myclass.module1_lib_id - - @pytest.mark.xfail def test_library_load_fail(): @@ -49,11 +33,3 @@ def test_library_load_fail(): """ pyfms.cfms.changelib(libpath="do_not_exist") - - -def test_remove_module(): - os.remove("module1.py") - - -if __name__ == "__main__": - test_write_module() From 48661513dff6fe6fdb06ec49919d58fc1425a5dd Mon Sep 17 00:00:00 2001 From: mlee03 Date: Thu, 29 May 2025 12:58:49 -0400 Subject: [PATCH 2/9] readme start --- README.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fdcb2a6..ea5b9f2 100644 --- a/README.md +++ b/README.md @@ -1 +1,12 @@ -# fms2py +# **`fms2py`** + +## **Installation** +`pyFMS` requires a compiled cFMS library. The library can be installed +by + +1. git clone --recursive https://github.com/NOAA-GFDL/pyFMS.git +2. cd pyFMS +3. python ./compile.py + +The script `compile.py` will first compile FMS within the cFMS submodule +directory From 1ec7599011eb31010890d19b56ebab59c9632cf8 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Thu, 29 May 2025 13:11:57 -0400 Subject: [PATCH 3/9] more readme --- README.md | 40 +++++++++++++++++++++++++++++++++++----- compile.py | 2 +- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index ea5b9f2..c979e1a 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,42 @@ # **`fms2py`** ## **Installation** -`pyFMS` requires a compiled cFMS library. The library can be installed -by +`pyFMS` requires a compiled cFMS library. If cFMS is not installed on the user's +local system, the library can be installed by +``` 1. git clone --recursive https://github.com/NOAA-GFDL/pyFMS.git 2. cd pyFMS -3. python ./compile.py +3. modify ./compile.py as instructed below +4. python ./compile.py +``` -The script `compile.py` will first compile FMS within the cFMS submodule -directory +The script `compile.py` will first compile the FMS library within the cFMS submodule +directory. Then, `compile.py` will compile the cFMS library linking to the just +compiled FMS library. By default, installations are located in `pyFMS/lib` directory. + +In order to compile cFMS, users will need to specify the following: + +1. Fortran and C compilers, for example, as shown below: + +``` +FC = "mpif90" +CC = "mpicc" +``` + +2. Path to the libyaml and netCDF installations, for example, as shown below: + +``` +yaml = "/opt/libyaml/0.2.5/GNU/14.2.0/" +netcdf = "/opt/netcdf/4.9.3/GNU/14.2.0/" +``` + +Upon `import pyfms` in your program, pyFMS will automatically load the cFMS library +in `pyFMS/lib`. If the cFMS library does not exist, or if you want to load your own +installation of cFMS, the following should be set before invoking any pyFMS methods: + +``` +import pyfms + +pyfms.cfms.init(libpath=path_to_cfms/libcFMS.so) +``` diff --git a/compile.py b/compile.py index 8e344fb..5bdb358 100644 --- a/compile.py +++ b/compile.py @@ -11,7 +11,7 @@ FC = "mpif90" #C compiler -CC ="mpicc" +CC = "mpicc" #default Fortran compiler flags FMS_FCFLAGS = f"-I{yaml}/include -I{netcdf}/include -fPIC" From 9bfd759e47d62eb9c1fa04bbbbf37270e37b514f Mon Sep 17 00:00:00 2001 From: mlee03 Date: Thu, 29 May 2025 13:13:41 -0400 Subject: [PATCH 4/9] more readme --- README.md | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index c979e1a..84dd9c0 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ # **`fms2py`** ## **Installation** -`pyFMS` requires a compiled cFMS library. If cFMS is not installed on the user's +pyFMS requires a compiled `cFMS` library. If `cFMS` is not installed on the user's local system, the library can be installed by ``` 1. git clone --recursive https://github.com/NOAA-GFDL/pyFMS.git 2. cd pyFMS -3. modify ./compile.py as instructed below +3. modify ./compile.py as instructed in Section compile.py 4. python ./compile.py ``` @@ -15,6 +15,17 @@ The script `compile.py` will first compile the FMS library within the cFMS submo directory. Then, `compile.py` will compile the cFMS library linking to the just compiled FMS library. By default, installations are located in `pyFMS/lib` directory. +Upon `import pyfms` in your program, pyFMS will automatically load the cFMS library +in `pyFMS/lib`. If the cFMS library does not exist, or if you want to load your own +installation of cFMS, the following should be set before invoking any pyFMS methods: + +``` +import pyfms + +pyfms.cfms.init(libpath=path_to_cfms/libcFMS.so) +``` + +## compile.py In order to compile cFMS, users will need to specify the following: 1. Fortran and C compilers, for example, as shown below: @@ -31,12 +42,3 @@ yaml = "/opt/libyaml/0.2.5/GNU/14.2.0/" netcdf = "/opt/netcdf/4.9.3/GNU/14.2.0/" ``` -Upon `import pyfms` in your program, pyFMS will automatically load the cFMS library -in `pyFMS/lib`. If the cFMS library does not exist, or if you want to load your own -installation of cFMS, the following should be set before invoking any pyFMS methods: - -``` -import pyfms - -pyfms.cfms.init(libpath=path_to_cfms/libcFMS.so) -``` From d7ca8a3fda3f5334e8279b316842c26107982370 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Thu, 29 May 2025 13:19:56 -0400 Subject: [PATCH 5/9] readme --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 84dd9c0..b9a5e92 100644 --- a/README.md +++ b/README.md @@ -7,17 +7,18 @@ local system, the library can be installed by ``` 1. git clone --recursive https://github.com/NOAA-GFDL/pyFMS.git 2. cd pyFMS -3. modify ./compile.py as instructed in Section compile.py +3. emacs ./compile.py (see Section compile.py) 4. python ./compile.py ``` -The script `compile.py` will first compile the FMS library within the cFMS submodule -directory. Then, `compile.py` will compile the cFMS library linking to the just -compiled FMS library. By default, installations are located in `pyFMS/lib` directory. +The script `compile.py` will first compile and install the FMS library within the +cFMS submodule directory to `pyfms/lib/FMS`. Then, compile.py will compile the cFMS library +linking to FMS in `pyfms/lib/FMS`. cFMS will be installed to `pyfms/lib/cFMS`. -Upon `import pyfms` in your program, pyFMS will automatically load the cFMS library -in `pyFMS/lib`. If the cFMS library does not exist, or if you want to load your own -installation of cFMS, the following should be set before invoking any pyFMS methods: +Upon `import pyfms`, pyFMS will automatically load the cFMS library +in `pyfms/lib/cFMS`. If the cFMS library does not exist, or if users want to load a +diferent instance of cFMS, the following should be set in the program before invoking +any pyFMS methods: ``` import pyfms From 52bf1874dd7d02b5e68991b4d59037db8328eb32 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Thu, 29 May 2025 13:23:14 -0400 Subject: [PATCH 6/9] readme --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b9a5e92..10575aa 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # **`fms2py`** ## **Installation** -pyFMS requires a compiled `cFMS` library. If `cFMS` is not installed on the user's +pyFMS requires a compiled `cFMS` library and the pyFMS repository contains +cFMS as a submodule. If `cFMS` is not installed on the user's local system, the library can be installed by ``` @@ -11,12 +12,12 @@ local system, the library can be installed by 4. python ./compile.py ``` -The script `compile.py` will first compile and install the FMS library within the -cFMS submodule directory to `pyfms/lib/FMS`. Then, compile.py will compile the cFMS library +The script `compile.py` will first compile and install the FMS library (which is +a submodule in cFMS) to `pyfms/lib/FMS`. Then, compile.py will compile the cFMS library linking to FMS in `pyfms/lib/FMS`. cFMS will be installed to `pyfms/lib/cFMS`. Upon `import pyfms`, pyFMS will automatically load the cFMS library -in `pyfms/lib/cFMS`. If the cFMS library does not exist, or if users want to load a +in `pyfms/lib/cFMS`. If the cFMS library does not exist, or if users wish to load a diferent instance of cFMS, the following should be set in the program before invoking any pyFMS methods: @@ -27,7 +28,8 @@ pyfms.cfms.init(libpath=path_to_cfms/libcFMS.so) ``` ## compile.py -In order to compile cFMS, users will need to specify the following: +In order to compile cFMS, users will need to specify the following +fields in the script `compile.py`: 1. Fortran and C compilers, for example, as shown below: From c5bd5c55b1328e01521be429c7ab9c7426276741 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Thu, 29 May 2025 13:24:09 -0400 Subject: [PATCH 7/9] readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 10575aa..453d850 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,8 @@ pyfms.cfms.init(libpath=path_to_cfms/libcFMS.so) ``` ## compile.py -In order to compile cFMS, users will need to specify the following -fields in the script `compile.py`: +To compile cFMS with the script `compile.py`, users will need to specify the following +fields: 1. Fortran and C compilers, for example, as shown below: From 3b6f5364e388078f9469354d5ff4e122a99c12f2 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Thu, 29 May 2025 13:25:48 -0400 Subject: [PATCH 8/9] by to with --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 453d850..56d0d85 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ## **Installation** pyFMS requires a compiled `cFMS` library and the pyFMS repository contains cFMS as a submodule. If `cFMS` is not installed on the user's -local system, the library can be installed by +local system, the library can be installed with ``` 1. git clone --recursive https://github.com/NOAA-GFDL/pyFMS.git From 78c2a6df6c71f68703e526364beeb6237e92fd8c Mon Sep 17 00:00:00 2001 From: mlee03 Date: Thu, 29 May 2025 13:40:56 -0400 Subject: [PATCH 9/9] include untracked file --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..0734326 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +graft pyfms/lib \ No newline at end of file