diff --git a/Coupled_Drivers/cice_driver.py b/Coupled_Drivers/cice_driver.py index bc0f1a1..dd08e7c 100644 --- a/Coupled_Drivers/cice_driver.py +++ b/Coupled_Drivers/cice_driver.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2023-2025 Met Office. All rights reserved. + (C) Crown copyright 2023-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -26,7 +26,6 @@ import re import datetime import time -import subprocess import time2days import inc_days import common @@ -34,6 +33,8 @@ import dr_env_lib.cice_def import dr_env_lib.env_lib +from mocilib import shellout + def __expand_array(short_array): ''' diff --git a/Coupled_Drivers/common.py b/Coupled_Drivers/common.py index d3470cf..43f56e9 100644 --- a/Coupled_Drivers/common.py +++ b/Coupled_Drivers/common.py @@ -296,6 +296,82 @@ def __exec_subproc_true_shell(cmd, verbose=True): return process.returncode, output +def exec_subproc_timeout(cmd, timeout_sec=10): + ''' + Execute a given shell command with a timeout. Takes a list containing + the commands to be run, and an integer timeout_sec for how long to + wait for the command to run. Returns the return code from the process + and the standard out from the command or 'None' if the command times out. + + This function is now DEPRECATED in favour of the exec_subprocess function + used in mocilib/shellout.py + ''' + process = subprocess.Popen(cmd, shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + timer = threading.Timer(timeout_sec, process.kill) + try: + timer.start() + stdout, err = process.communicate() + if err: + sys.stderr.write('[SUBPROCESS ERROR] %s\n' % error) + rcode = process.returncode + finally: + timer.cancel() + if sys.version_info[0] >= 3: + output = stdout.decode() + else: + output = stdout + return rcode, output + + +def exec_subproc(cmd, verbose=True): + ''' + Execute given shell command. Takes a list containing the commands to be + run, and a logical verbose which if set to true will write the output of + the command to stdout. + + This function is now DEPRECATED in favour of the exec_subprocess function + used in mocilib/shellout.py + ''' + process = subprocess.Popen(cmd, shell=False, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output, err = process.communicate() + if verbose and output: + sys.stdout.write('[SUBPROCESS OUTPUT] %s\n' % output) + if err: + sys.stderr.write('[SUBPROCESS ERROR] %s\n' % error) + if sys.version_info[0] >= 3: + output = output.decode() + return process.returncode, output + + +def __exec_subproc_true_shell(cmd, verbose=True): + ''' + Execute given shell command, with shell=True. Only use this function if + exec_subproc does not work correctly. Takes a list containing the commands + to be run, and a logical verbose which if set to true will write the + output of the command to stdout. + + This function is now DEPRECATED in favour of the exec_subprocess function + used in mocilib/shellout.py + ''' + process = subprocess.Popen(cmd, shell=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output, err = process.communicate() + if verbose and output: + sys.stdout.write('[SUBPROCESS OUTPUT] %s\n' % output) + if err: + sys.stderr.write('[SUBPROCESS ERROR] %s\n' % error) + if sys.version_info[0] >= 3: + output = output.decode() + return process.returncode, output + def _calculate_ppn_values(nproc, nodes): ''' Calculates number of processes per node and numa node for launch diff --git a/Coupled_Drivers/cpmip_utils.py b/Coupled_Drivers/cpmip_utils.py index f6bfcf2..8ff0e18 100644 --- a/Coupled_Drivers/cpmip_utils.py +++ b/Coupled_Drivers/cpmip_utils.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2023-2025 Met Office. All rights reserved. + (C) Crown copyright 2023-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy diff --git a/Coupled_Drivers/cpmip_xios.py b/Coupled_Drivers/cpmip_xios.py index d9dc65a..4a02c2c 100644 --- a/Coupled_Drivers/cpmip_xios.py +++ b/Coupled_Drivers/cpmip_xios.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy diff --git a/Coupled_Drivers/mct_driver.py b/Coupled_Drivers/mct_driver.py index 0b02996..17c9bd7 100644 --- a/Coupled_Drivers/mct_driver.py +++ b/Coupled_Drivers/mct_driver.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy of the code, the use, duplication or disclosure of it is strictly @@ -28,6 +28,9 @@ import dr_env_lib.mct_def import dr_env_lib.env_lib import cpmip_controller + +from mocilib import shellout + try: import f90nml except ImportError: diff --git a/Coupled_Drivers/nemo_driver.py b/Coupled_Drivers/nemo_driver.py index e816772..63d4359 100644 --- a/Coupled_Drivers/nemo_driver.py +++ b/Coupled_Drivers/nemo_driver.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2023-2025 Met Office. All rights reserved. + (C) Crown copyright 2023-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -29,6 +29,8 @@ import common import error +from mocilib import shellout + try: import cf_units except ImportError: diff --git a/Coupled_Drivers/rivers_driver.py b/Coupled_Drivers/rivers_driver.py index 8dfab29..7af4864 100644 --- a/Coupled_Drivers/rivers_driver.py +++ b/Coupled_Drivers/rivers_driver.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2025 Met Office. All rights reserved. + (C) Crown copyright 2025-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -26,6 +26,8 @@ import error import dr_env_lib.rivers_def import dr_env_lib.env_lib + +from mocilib import shellout try: import f90nml except ImportError: diff --git a/Coupled_Drivers/si3_controller.py b/Coupled_Drivers/si3_controller.py index b0a93e2..8394f36 100644 --- a/Coupled_Drivers/si3_controller.py +++ b/Coupled_Drivers/si3_controller.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy of the code, the use, duplication or disclosure of it is strictly @@ -26,6 +26,8 @@ import dr_env_lib.ocn_cont_def import dr_env_lib.env_lib +from mocilib import shellout + def _check_si3nl_envar(envar_container): ''' Get the si3 namelist file exists diff --git a/Coupled_Drivers/top_controller.py b/Coupled_Drivers/top_controller.py index f1d2478..ce40779 100644 --- a/Coupled_Drivers/top_controller.py +++ b/Coupled_Drivers/top_controller.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -73,6 +73,8 @@ import dr_env_lib.ocn_cont_def import dr_env_lib.env_lib +from mocilib import shellout + # Define errors for the TOP controller only SERIAL_MODE_ERROR = 99 diff --git a/Coupled_Drivers/unittests/test_cpmip_utils.py b/Coupled_Drivers/unittests/test_cpmip_utils.py index 964b97c..6121f93 100644 --- a/Coupled_Drivers/unittests/test_cpmip_utils.py +++ b/Coupled_Drivers/unittests/test_cpmip_utils.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2023-2025 Met Office. All rights reserved. + (C) Crown copyright 2023-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy diff --git a/Coupled_Drivers/unittests/test_cpmip_xios.py b/Coupled_Drivers/unittests/test_cpmip_xios.py index 2d1c087..b9b62dd 100644 --- a/Coupled_Drivers/unittests/test_cpmip_xios.py +++ b/Coupled_Drivers/unittests/test_cpmip_xios.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy diff --git a/Coupled_Drivers/unittests/test_rivers_driver.py b/Coupled_Drivers/unittests/test_rivers_driver.py index 5d9a206..1ee2cd6 100644 --- a/Coupled_Drivers/unittests/test_rivers_driver.py +++ b/Coupled_Drivers/unittests/test_rivers_driver.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2025 Met Office. All rights reserved. + (C) Crown copyright 2025-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy diff --git a/Coupled_Drivers/write_namcouple.py b/Coupled_Drivers/write_namcouple.py index 5d1fb85..1273e42 100644 --- a/Coupled_Drivers/write_namcouple.py +++ b/Coupled_Drivers/write_namcouple.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy of the code, the use, duplication or disclosure of it is strictly @@ -25,6 +25,8 @@ import write_namcouple_fields import write_namcouple_header +from mocilib import shellout + # Dictionary containing the RMP mappings RMP_MAPPING = {'Bc':'BICUBIC', 'Bi':'BILINEA', diff --git a/Coupled_Drivers/xios_driver.py b/Coupled_Drivers/xios_driver.py index 3834673..ad11631 100644 --- a/Coupled_Drivers/xios_driver.py +++ b/Coupled_Drivers/xios_driver.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2023-2025 Met Office. All rights reserved. + (C) Crown copyright 2023-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -27,6 +27,8 @@ import dr_env_lib.xios_def import dr_env_lib.env_lib +from mocilib import shellout + def _copy_iodef_custom(xios_evar): ''' If a custom iodef file exists, copy this to the required input filename diff --git a/Postprocessing/common/utils.py b/Postprocessing/common/utils.py index 9a79af4..a27c675 100644 --- a/Postprocessing/common/utils.py +++ b/Postprocessing/common/utils.py @@ -26,6 +26,8 @@ import subprocess import timer +from mocilib import shellout + globals()['debug_mode'] = None globals()['debug_ok'] = True diff --git a/Postprocessing/unittests/test_shellout.py b/Postprocessing/unittests/test_shellout.py new file mode 100644 index 0000000..a914c62 --- /dev/null +++ b/Postprocessing/unittests/test_shellout.py @@ -0,0 +1,33 @@ +# ----------------------------------------------------------------------------- +# (C) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +# ----------------------------------------------------------------------------- + +import unittest +from mocilib import shellout +from hypothesis import given, strategies as st + +class ExecTets(unittest.TestCase): + ''' Unit tests for executing shellout commands''' + + def test_semicolon_commands(self): + cmd = "echo Hello There;echo General Kenobi" + _,rcode = shellout._exec_subprocess(cmd=cmd) + assert rcode == 0 + + def test_and_commands(self): + cmd ="echo Hello There&&echo General Kenobi" + _,rcode = shellout._exec_subprocess(cmd=cmd) + assert rcode == 0 + + @given(st.text()) + def test_called_process_error(self,directory): + cmd = f"ls /{directory}" + _,rcode = shellout._exec_subprocess(cmd=cmd) + assert rcode != 0 + + def test_timeout_expired(self): + cmd = "sleep 15" + _,rcode = shellout._exec_subprocess(cmd=cmd,timeout=10) + assert rcode != 0 diff --git a/mocilib/__init__.py b/mocilib/__init__.py new file mode 100644 index 0000000..ecf7eac --- /dev/null +++ b/mocilib/__init__.py @@ -0,0 +1,2 @@ +___all__ = ["shellout"] +from . import shellout diff --git a/mocilib/shellout.py b/mocilib/shellout.py new file mode 100644 index 0000000..7e3afd4 --- /dev/null +++ b/mocilib/shellout.py @@ -0,0 +1,51 @@ +# ----------------------------------------------------------------------------- +# (C) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +# ----------------------------------------------------------------------------- + +import subprocess +import os +import sys +import shlex + + +def _exec_subprocess(cmd, verbose=False, timeout=None ,current_working_directory=os.getcwd()): + """ + Execute a given shell command + + :param cmd: The command to be executed given as a string + :param verbose: A boolean value to determine if the stdout + stream is displayed during the runtime. + :param current_working_directory: The directory in which the + command should be executed. + """ + + cmd = shlex.split(cmd) + + try: + + output = subprocess.run( + cmd, + capture_output=True, + cwd=current_working_directory, + timeout=timeout, + check=True + ) + rcode = output.returncode + output_message = output.stdout.decode() + + if verbose and output: + sys.stdout.write(f"[DEBUG]{output.stdout}\n") + if output.stderr and output.returncode != 0: + sys.stderr.write(f"[ERROR] {output.stderr}\n") + + except subprocess.CalledProcessError as exc: + output_message = exc.stdout.decode() if exc.stdout else "" + rcode = exc.returncode + + except subprocess.TimeoutExpired as exc: + output_message = exc.stdout.decode() if exc.stdout else "" + rcode = exc.returncode + + return rcode,output_message