From 312fdd5a161bdf77c6ed890232eec61e53bb7ec1 Mon Sep 17 00:00:00 2001 From: James Bruten <109733895+james-bruten-mo@users.noreply.github.com> Date: Tue, 4 Nov 2025 13:20:25 +0000 Subject: [PATCH 1/9] UKCA test suite port to git (#1) Co-authored-by: Erica Neininger <107684099+ericaneininger@users.noreply.github.com> --- dependencies.yaml | 26 + .../bin/script_updater.sh | 36 - .../app/export_simsys_scripts/rose-app.conf | 2 - rose-stem/app/extract_source/rose-app.conf | 8 + .../bin/rosestem_branch_checker.py | 140 -- rose-stem/app/umdp3_checker/rose-app.conf | 19 +- rose-stem/bin/ampersands.py | 477 ------- rose-stem/bin/fstring_parse.py | 535 -------- rose-stem/bin/indentation.py | 349 ----- rose-stem/bin/styling.py | 1141 ----------------- rose-stem/bin/umdp3_fixer.py | 408 ------ rose-stem/bin/whitespace.py | 275 ---- rose-stem/flow.cylc | 47 +- rose-stem/include/meto/graph.rc | 3 +- rose-stem/include/meto/queues.rc | 4 - rose-stem/include/meto/runtime.rc | 6 + rose-stem/include/runtime.rc | 37 +- rose-stem/include/vm/graph.rc | 2 +- rose-stem/include/vm/runtime.rc | 5 + rose-stem/lib/python/read_sources.py | 60 + rose-stem/lib/python/utils.py | 11 + rose-stem/rose-suite.conf | 6 +- 22 files changed, 204 insertions(+), 3393 deletions(-) create mode 100644 dependencies.yaml delete mode 100755 rose-stem/app/export_simsys_scripts/bin/script_updater.sh delete mode 100644 rose-stem/app/export_simsys_scripts/rose-app.conf create mode 100644 rose-stem/app/extract_source/rose-app.conf delete mode 100755 rose-stem/app/umdp3_checker/bin/rosestem_branch_checker.py delete mode 100755 rose-stem/bin/ampersands.py delete mode 100644 rose-stem/bin/fstring_parse.py delete mode 100755 rose-stem/bin/indentation.py delete mode 100755 rose-stem/bin/styling.py delete mode 100755 rose-stem/bin/umdp3_fixer.py delete mode 100755 rose-stem/bin/whitespace.py create mode 100644 rose-stem/lib/python/read_sources.py create mode 100644 rose-stem/lib/python/utils.py diff --git a/dependencies.yaml b/dependencies.yaml new file mode 100644 index 0000000..ad0b21d --- /dev/null +++ b/dependencies.yaml @@ -0,0 +1,26 @@ +############################################################################## +# (c) Crown copyright 2024 Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## + +# This file contains a list of all linked external repositories containing code +# and data required to run the entire test suite +# +# Each section is the name of the repository +# source: Is either the url to the repository (the upstream or +# a fork) or a path to a local clone with format `host:/path/to/clone` +# ref: Is a valid git reference. This can either be a commit hash, a tag or a +# branch name +# +# The entry for this repository will have blank values for both source and ref +# by default, which will cause the test suite to use the source code from the +# local clone. These can be filled to set a different source if desired. + +SimSys_Scripts: + source: git@github.com:MetOffice/SimSys_Scripts.git + ref: main + +ukca: + source: + ref: diff --git a/rose-stem/app/export_simsys_scripts/bin/script_updater.sh b/rose-stem/app/export_simsys_scripts/bin/script_updater.sh deleted file mode 100755 index 73f8a5e..0000000 --- a/rose-stem/app/export_simsys_scripts/bin/script_updater.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# *****************************COPYRIGHT******************************* -# (C) Crown copyright Met Office. All rights reserved. -# For further details please refer to the file COPYRIGHT.txt -# which you should have received as part of this distribution. -# *****************************COPYRIGHT******************************* - -github_url=${github_url:-"https://github.com/MetOffice/SimSys_Scripts.git"} -github_branch=${github_branch:-"main"} - -echo "Git-scripts updater has started running" - -rm -rf "${CYLC_SUITE_SHARE_DIR}/imported_github_scripts" -if [[ $? != 0 ]]; then - echo "Couldn't remove specified folder. Try checking permissions" - exit 1 - else - echo "Successfully removed old SimSys_Scripts git directory" -fi - -git clone -b "$github_branch" --single-branch "$github_url" \ - "${CYLC_SUITE_SHARE_DIR}/imported_github_scripts" 2>&1 -if [[ $? != 0 ]]; then - echo "Unable to clone remote git repo into specified location." - echo "Check git branch, git url, destination path, permissions and git access" - exit 1 - else - echo "Git repo successfully cloned" -fi - -echo "Copying suite_report and fcm_bdiff to cylc-run/bin" -cp "${CYLC_SUITE_SHARE_DIR}/imported_github_scripts/suite_report.py" "${CYLC_SUITE_RUN_DIR}/bin" -cp "${CYLC_SUITE_SHARE_DIR}/imported_github_scripts/fcm_bdiff.py" "${CYLC_SUITE_RUN_DIR}/bin" - -echo "Github scripts updated" diff --git a/rose-stem/app/export_simsys_scripts/rose-app.conf b/rose-stem/app/export_simsys_scripts/rose-app.conf deleted file mode 100644 index 4446104..0000000 --- a/rose-stem/app/export_simsys_scripts/rose-app.conf +++ /dev/null @@ -1,2 +0,0 @@ -[command] -default=script_updater.sh diff --git a/rose-stem/app/extract_source/rose-app.conf b/rose-stem/app/extract_source/rose-app.conf new file mode 100644 index 0000000..90364b5 --- /dev/null +++ b/rose-stem/app/extract_source/rose-app.conf @@ -0,0 +1,8 @@ +[command] +default=python rose_stem_extract_source.py + +[file:get_git_sources.py] +source=git:https://github.com/MetOffice/SimSys_Scripts.git::github_scripts/get_git_sources.py::main + +[file:rose_stem_extract_source.py] +source=git:https://github.com/MetOffice/SimSys_Scripts.git::github_scripts/rose_stem_extract_source.py::main diff --git a/rose-stem/app/umdp3_checker/bin/rosestem_branch_checker.py b/rose-stem/app/umdp3_checker/bin/rosestem_branch_checker.py deleted file mode 100755 index 798b7fe..0000000 --- a/rose-stem/app/umdp3_checker/bin/rosestem_branch_checker.py +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env python3 -# -# *****************************COPYRIGHT******************************* -# (C) Crown copyright Met Office. All rights reserved. -# For further details please refer to the file COPYRIGHT.txt -# which you should have received as part of this distribution. -# *****************************COPYRIGHT******************************* - -''' -Rose-stem test for checking the branch working copy with the trunk. - -Fail if any files are changed by the umdp3_checker.py script. - -Return a list of any files changed by the umdp3_checker.py script. - -Usage: - Fortran files must end in .f90, .F90 or .inc extension. - Copy files to work/tmp. - Run the script on the whole tmp dir. - Diff with the working copy. - Fail if changes and return files that need changing. - -''' - -from optparse import OptionParser -import os -import shutil -import subprocess -import tempfile - - -def copy_working_branch(model_source): - '''Copy the working version of the branch to a tmp dir.''' - # Make the tmp dir in the cwd which is the work/ of the task name. - tmpdir = tempfile.mkdtemp() - tmp_filename = 'model_diff' - - # Ensure the file is read/write by the creator only - saved_umask = os.umask(0o077) - - tmp_path = os.path.join(tmpdir, tmp_filename) - - # Copy the src/ from the working copy to the tmp dir. - - src_wkcopy = os.path.join(model_source, "src") - try: - shutil.copytree(src_wkcopy, tmp_path) - # Directories are the same - except shutil.Error as err: - print('Directory not copied. Error: %s' % err) - # Any error saying that the directory doesn't exist - except OSError as err: - print('Directory not copied. Error: %s' % err) - return tmp_path, saved_umask, tmpdir - - -def diff_cwd_working(model_source, path, saved_umask, tmpdir): - '''Diff the tmp dir with the working branch and report diff.''' - diff = subprocess.run("diff -qr " + model_source + "/src " + path, - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True, - universal_newlines=True) - - if diff.returncode == 0: - print("[OK] No changes were made by the UMDP3 checker script and " - "the working copy complies with the coding standards. " - "No action required.") - else: - diff_stdout = diff.stdout - diff_files = diff_stdout.strip().split("\n") - print("[FAIL] The following files were changed when the " - "umdp3_fixer.py script was run:") - for diff_filesname in diff_files: - # diffs are of the form "Files and differ" - # we select only - print("[FAIL] " + diff_filesname.split()[1]) - print("Please run rose-stem/bin/umdp3_fixer.py on each of the " - "failed files in your working copy and check the changes. " - "Then commit the changes to your branch and then re-run all " - "rose-stem testing.") - os.umask(saved_umask) - shutil.rmtree(tmpdir) - raise ValueError("Ran fcm diff command and changes were made by " + - "the umdp3_fixer.py script.") - - return - - -def run_umdp3checker(model_source, path, amp_column): - '''Run the umdp3 fixer script in the tmp dir copy of the working branch.''' - umdp3 = subprocess.run(model_source + "/rose-stem/bin/umdp3_fixer.py " + - "--col {0:} ".format(amp_column) + - "$(find " + path + - " -name '*.[F|f]90' -o -name '*.inc' | xargs)", - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - shell=True) - - if umdp3.returncode != 0: - print("[FAIL] Problem while attempting to run umdp3_fixer.py") - raise ValueError("Problem while attempting to run umdp3_fixer.py") - return - - -def main(): - '''Take in the location of working branch location and run the tests.''' - # Initialise the command line parser. - description = 'Args for the source code...' - parser = OptionParser(description=description) - parser.add_option('--source', - dest='source', - action='store', - help='source of the model branch', - default='None') - # e.g. "--source model_source_branch" - - parser.add_option('--col', - dest='col', - action='store', - help='Column to put "&"s in', - type="int", - default='80') - # e.g. "--col 80" - - # Parse the command line. - (opts, _) = parser.parse_args() - model_source = opts.source - amp_column = opts.col - - (path, saved_umask, tmpdir) = copy_working_branch(model_source) - run_umdp3checker(model_source, path, amp_column) - diff_cwd_working(model_source, path, saved_umask, tmpdir) - - os.umask(saved_umask) - shutil.rmtree(tmpdir) - -if __name__ == "__main__": - main() diff --git a/rose-stem/app/umdp3_checker/rose-app.conf b/rose-stem/app/umdp3_checker/rose-app.conf index e27ec0c..e94409f 100644 --- a/rose-stem/app/umdp3_checker/rose-app.conf +++ b/rose-stem/app/umdp3_checker/rose-app.conf @@ -1,24 +1,27 @@ [command] -default=rosestem_branch_checker.py --source "$PWD" --col 80 +default=python3 rosestem_branch_checker.py --source "$PWD" --col 80 + +[file:rosestem_branch_checker.py] +source=$SOURCE_DIRECTORY/SimSys_Scripts/umdp3_fixer/rosestem_branch_checker.py [file:rose-stem/bin/ampersands.py] -source=$HOST_SOURCE_UKCA_BASE/rose-stem/bin/ampersands.py +source=$SOURCE_DIRECTORY/SimSys_Scripts/umdp3_fixer/ampersands.py [file:rose-stem/bin/fstring_parse.py] -source=$HOST_SOURCE_UKCA_BASE/rose-stem/bin/fstring_parse.py +source=$SOURCE_DIRECTORY/SimSys_Scripts/umdp3_fixer/fstring_parse.py [file:rose-stem/bin/indentation.py] -source=$HOST_SOURCE_UKCA_BASE/rose-stem/bin/indentation.py +source=$SOURCE_DIRECTORY/SimSys_Scripts/umdp3_fixer/indentation.py [file:rose-stem/bin/styling.py] -source=$HOST_SOURCE_UKCA_BASE/rose-stem/bin/styling.py +source=$SOURCE_DIRECTORY/SimSys_Scripts/umdp3_fixer/styling.py [file:rose-stem/bin/umdp3_fixer.py] -source=$HOST_SOURCE_UKCA_BASE/rose-stem/bin/umdp3_fixer.py +source=$SOURCE_DIRECTORY/SimSys_Scripts/umdp3_fixer/umdp3_fixer.py [file:rose-stem/bin/whitespace.py] -source=$HOST_SOURCE_UKCA_BASE/rose-stem/bin/whitespace.py +source=$SOURCE_DIRECTORY/SimSys_Scripts/umdp3_fixer/whitespace.py [file:src] -source=$HOST_SOURCE_UKCA_BASE/src +source=$SOURCE_DIRECTORY/ukca/src diff --git a/rose-stem/bin/ampersands.py b/rose-stem/bin/ampersands.py deleted file mode 100755 index d8f960f..0000000 --- a/rose-stem/bin/ampersands.py +++ /dev/null @@ -1,477 +0,0 @@ -#!/usr/bin/env python3 -# *****************************COPYRIGHT******************************* -# (C) Crown copyright Met Office. All rights reserved. -# For further details please refer to the file COPYRIGHT.txt -# which you should have received as part of this distribution. -# *****************************COPYRIGHT******************************* -''' -## NOTE ## -This module is one of several for which the Master copy is in the -UM repository. When making changes, please ensure the changes are made in the UM -repository or they will be lost during the release process when the UM copy is -copied over. - -This module contains various functions for putting continuation ampersands in -the same column throughout FORTRAN source code. - -If there are comments after the continuation ampersand, then the whitespace is -adjusted as much as possible to reduce line to the required length. - -Lines that are still too long will be identified, optionally with a message in -stdout. - -Note that this code has made an effort to deal with the possibility of -amersands and exclamation marks within quoted strings or comments, but there -may be some cases which are missed. These lines will be left without applying -the ampersand shifting, and will be flagged, optionally with a message in -stdout. -''' -import sys -import re -import traceback -from optparse import OptionParser -from fstring_parse import * - -# Default column in which to place ampersands. -DEFAULT_COL = 80 - - -class CharError(ParsingError): - ''' - Raised when there are an unexpected number of a certain char in a line. - ''' - def __init__(self, char, number): - self.number = number - self.char = char - self.msg = "There are {0:d} unquoted, uncommented " \ - "\"{1:s}\" in this line.".format(number, char) - pass - - -def print_message(errtype, msg, iline=None, line=None, fname=None): - ''' - Print a formatted message - ''' - if fname is None: - fnamestr = "" - else: - fnamestr = "{0:s}:".format(fname) - - if iline is None: - if fnamestr is None: - ilinestr = "" - else: - ilinestr = " " - else: - ilinestr = "L{0:05d}: ".format(iline) - - if line is None: - linestr = "" - else: - linestr = ": {0:s}".format(line) - - print("{0:s}{1:s}{2:s} - {3:s}{4:s}".format(fnamestr, ilinestr, errtype, - msg, linestr)) - - -def shift_ampersand(line, line_previous, str_continuation, col=DEFAULT_COL, - preclean=False): - ''' - Check if the line contains an ampersand. - If so then set location of ampersand to col so as to be consistent - Sometimes there are comments after the ampersand, in this case keep - the comment and ampersand where they are, unless the line is too long, in - which case reduce any whitespace between the ampersand and the - comment. If the line is still too long, then reduce whitespace between - the end of the code and the ampersand until the comment fits within the - required line length. - ''' - - # return earliy if there are no apersands at all. - if "&" not in line: - return line - - if (len(re.findall("&", line)) < 2) and preclean is True: - return line - - lp = len(line_previous) - - if lp > 0: - workline = re.sub(r"\\(\s*)$", r" \1", line_previous) + line - else: - workline = line - - col = col + lp - - stripline = workline.strip() - - # Pre-processor lines start with #. Ignore them completely. - pre_proc = (stripline[0] == "#") - - # Lines that are completely commented start with a bang and are also - # ignored completely (except if they are actually OpenMP) - all_comment = (stripline[0] == "!") - omp_sentinal = all_comment and (stripline[1] == "$") - - # Ignore empty lines or pre-processor directives - - if len(line.strip()) == 0 or pre_proc: - return line - - # Ignore whole-line comments. - if all_comment and not omp_sentinal: - return line - - # First need to make sure that the continuation character and any - # comments are found correctly. Can't just search for the - # first ampersand or bang in the line as it is possible that strings - # (text inside single or double quotes) and comments could contain - # ampersands or bangs, and these must be ignored when processing the - # line. - # The solution is to first find any possibly misleading characters - # and replace them with temporary substitutes. Then find the location - # of any continuation character and/or comment in the line, before - # replacing the substituted characters, and the process the line to - # correctly align it. - # The order of substitutions is - # 1. Find any ampersands that are within pairs of double or single - # quotes and replace them with "X". - # e.g. - # CALL log_info("init_veg", "Doell & Siebert", & - # becomes - # CALL log_info("init_veg", "Doell X Siebert", & - # 2. Find any bangs that are within pairs of double or single quotes - # and replace them with "Y". - # e.g. - # CALL log_info("init_veg", "Doell method!") ! Describe method - # becomes - # CALL log_info("init_veg", "Doell methodY") ! Describe method - # 3. Find any ampersands that are within comments and replace them - # with "Z". - # e.g. - # CALL log_info("init_veg", "Doell-Siebert", & ! Doell & Siebert - # becomes - # CALL log_info("init_veg", "Doell-Siebert", & ! Doell Z Siebert - # Once these replacements have been done, the line should only contain - # at most one ampersand (as continuation character), and if it contains - # any bangs, the first one is the start of the comment. Then find - # the location of these to start any required processing. - - # Find and replace any OpenMP sentinals - - omp_loc = workline.find("!$") - - if omp_loc != -1: - omp_continue = workline.upper().find("!$OMP&") - if omp_continue != -1: - workline = replace_characters(workline, [omp_continue+5], [1], - replchar=" ") - workline = replace_characters(workline, [omp_loc], [1], replchar="W") - - # Find where there are ampersands or bangs within single or double - # quotes. - - quoted_amp_locs = find_quoted_char(workline, "&", str_continuation) - quoted_bang_locs = find_quoted_char(workline, "!", str_continuation) - - # If any ampersands or bangs were found within quotes, replace them - # with "X" and "Y" respectively. - if quoted_amp_locs is not None: - lens = len(quoted_amp_locs) * [1, ] - workline = replace_characters(workline, quoted_amp_locs, lens, - replchar="X") - if quoted_bang_locs is not None: - lens = len(quoted_bang_locs) * [1, ] - workline = replace_characters(workline, quoted_bang_locs, lens, - replchar="Y") - - # Find where there are ampersands within comments and replace them with - # "Z" temporarily. - commented_amp_locs = find_commented_char(workline, "&", str_continuation) - - if commented_amp_locs is not None: - lens = len(commented_amp_locs) * [1, ] - workline = replace_characters(workline, commented_amp_locs, lens, - replchar="Z") - - # Check if there is still more than one ampersand in this line and - # warn if there is. - if (len(re.findall("&", workline)) > 1): - amp_loc = workline.find("&") - - # determin if there is a leading ampersand - beforeline = workline[lp:amp_loc].rstrip() - - if len(beforeline) == 0: - # the first ampersand is a leading continuation - workline = replace_characters(workline, [amp_loc], [1], - replchar=" ") - else: - raise CharError("&", len(re.findall("&", workline))) - - finder = workline.find - - # Find the first ampersand in the line (if there is one). - amp_loc = finder("&") - - # Find the first bang in the line to see if it is a comment or OpenMP. - comment_loc = finder("!") - - # Now the locations of the characters that are needed have been found, - # replace ampersands and bangs where they were. - if quoted_amp_locs is not None: - lens = len(quoted_amp_locs) * [1, ] - workline = replace_characters(workline, quoted_amp_locs, lens, - replchar="&") - if quoted_bang_locs is not None: - lens = len(quoted_bang_locs) * [1, ] - workline = replace_characters(workline, quoted_bang_locs, lens, - replchar="!") - if commented_amp_locs is not None: - lens = len(commented_amp_locs) * [1, ] - workline = replace_characters(workline, commented_amp_locs, lens, - replchar="&") - if omp_loc != -1: - workline = replace_characters(workline, [omp_loc], [1], replchar="!") - - if preclean is True: - return workline[lp:] - - # If the line contains an unquoted or uncommented ampersand, need - # to check if it is in the right place. - if amp_loc != -1: - - # If there is no comment, determin if it is a leading continuation - # ampersand. If it is remove it; else shift the ampersand (and remove - # any trailing whitespace). - if comment_loc == -1: - - # determin if we are a leading ampersand - beforeline = workline[lp:amp_loc].rstrip() - - if len(beforeline) == 0: - # the ampersand is a leading continuations - workline = replace_characters(workline, [amp_loc], [1], - replchar=" ") - else: - # Keep the part of the input line before the ampersand (without - # white space). - workline = workline[:amp_loc].rstrip() - temp_length = len(workline) - - # Don't require a space between the end of the code and the - # continuation character, so maximum length of the code is - # col - 1. - if temp_length > col - 1: - # If the line is too long, just add the ampersand at the - # end (after a space). - workline = list(workline) - workline.append(" &") - workline = "".join(workline) - else: - # Otherwise, add the text to an empty line with an - # ampersand at the end. - templine = list(" ") * col - templine[-1] = "&" - workline = list(workline) - templine[0:temp_length] = workline[0:temp_length] - workline = "".join(templine) - - # If there is a comment after the ampersand then only do anything - # if the line is too long, in which case can try to take white - # space to get it down to length. - elif comment_loc > amp_loc: - - # If the line is too long, first make sure there is only one - # space between ampersand and comment, and get rid of any - # trailing whitespace. - if len(workline.rstrip()) > col: - comment = workline[comment_loc:].rstrip() - workline = workline[:amp_loc+1] - workline = " ".join([workline, comment]) - - # If the line is still too long, see if ampersand can be moved - # left to reduce the length sufficiently. This can only be done - # if the line doesn't end in a string continuation - str_cont_test = is_str_continuation(workline, str_continuation) - check_quote = str_cont_test[SQUOTE] or str_cont_test[DQUOTE] - str_size_test = len(workline.rstrip()) > col - if str_size_test and not check_quote: - # How much do we need to reduce the length to get it short - # enough? - nloop = len(workline) - col - - # Get the current location of the ampersand. - amp_location = amp_loc - - # Cast the line to a list as characters can't be edited in - # place in a string. - workline = list(workline) - - # Try cutting down the whitespace one step at a time until - # there is only one space left. - for i in range(nloop): - - if workline[amp_location-1] == " ": - # If there is still whitespace that can be removed, - # remove it and update the ampersand location. - del workline[amp_location-1] - amp_location -= 1 - else: - # Ampersand is now next to no-blank text so place - # a single space and exit the loop. - workline.insert(amp_location, " ") - break - - # Join the line back up into a string. - workline = "".join(workline) - - return workline[lp:] - - -def check_line_len(line, maxlinelen=DEFAULT_COL): - ''' - Check line to see if it violates length requirements. If debugging, - write some information to stdout. - ''' - - return (len(line) > maxlinelen) - - -def apply_ampersand_shift(lines, col=DEFAULT_COL, fname=None, debug=False, - preclean=False): - ''' - For a lot of lines make sure any continuation ampersands are in the - same column and return the result - ''' - - not_parsed = [] - output_lines = [] - continuation = False - pp_continuation = False - str_continuation = [False, False] - line_previous = "" - - for iline, line in enumerate(lines): - try: - outline = shift_ampersand(line, line_previous, str_continuation, - col, preclean) - except ParsingError as e: - if debug: - print_message("PARSING ERROR", - "{0:s} Ampersand shifting has not been " - "applied".format(e.msg), iline+1, line=line, - fname=fname) - outline = line - not_parsed.append(iline) - - # Check for the start of new pre-processor continuation - if is_pp_continuation(line) and not pp_continuation: - pp_continuation = True - - # test the line continuation properties of this line - if pp_continuation: - if not is_pp_continuation(line): - pp_continuation = False - line_previous = "" - elif continuation: - if is_continuation(line, str_continuation): - # check if still string continuation - str_continuation = is_str_continuation(line, str_continuation) - else: - continuation = False - str_continuation = [False, False] - else: - # Finally, detect if the following line is a continuation - # of this one (and therefore requires no indentation) - if is_continuation(line, str_continuation): - continuation = True - str_continuation = is_str_continuation(line, str_continuation) - - # if we are a pp continuation, save the partial line - if pp_continuation: - line_previous = re.sub(r"\\\s*$", "", line_previous) + line - - output_lines.append(outline) - - return output_lines, not_parsed - - -def apply_check_line_len(lines, fname=None, maxlinelen=DEFAULT_COL, - debug=False): - ''' - For a lot of lines check if any lines are longer than required - ''' - - any_too_long = False - ilines_too_long = [] - for iline, line in enumerate(lines): - iline_too_long = check_line_len(line, maxlinelen) - if iline_too_long: - any_too_long = True - ilines_too_long.append(iline) - if debug: - print_message("VIOLATION", - "Line > {0:d} columns".format(maxlinelen), - iline+1, line=line, fname=fname) - - if any_too_long: - return ilines_too_long - else: - return None - - -def main(): - ''' - Main toplevel function for testing - ''' - parser = OptionParser(usage=""" - %prog [--column col] [--debug] file_1 [file_2 [file_3] ... ] - - This script will attempt to manipulate white space to make sure ampersands - are consistently in the same column in each line. - - If the line is still too long, it will minimise its length. - - The optional --column tells which column should be used (default=80) - """) - parser.add_option("--column", dest="col", type="int", default=DEFAULT_COL, - help="Column in which ampersands should appear") - parser.add_option("--debug", action="store_true", - help="Report useful information") - - (opts, args) = parser.parse_args() - - input_file = args[0] - - if opts.debug: - print_message("INFO", "Debug mode is on.") - - with open(input_file, "r+") as file_in: - lines_in = file_in.read().split("\n") - new_lines, not_parsed = apply_ampersand_shift(lines_in, opts.col, - fname=input_file, - debug=opts.debug) - if opts.debug: - if len(not_parsed) > 0: - print_message("WARNING", - "Ampersand alignment failed for some lines " - "due to parsing errors. Please check lines and " - "make sure they are correct.", - fname=input_file) - ilines_too_long = apply_check_line_len(new_lines, input_file, - maxlinelen=opts.col, - debug=True) - if ilines_too_long is not None: - print_message("WARNING", - "Some lines are longer than {0:d} characters. " - "Please check and make them " - "shorter.".format(opts.col), fname=input_file) - - file_in.seek(0) - file_in.write("\n".join(new_lines)) - file_in.truncate() - - -if __name__ == "__main__": - main() diff --git a/rose-stem/bin/fstring_parse.py b/rose-stem/bin/fstring_parse.py deleted file mode 100644 index cc2bd60..0000000 --- a/rose-stem/bin/fstring_parse.py +++ /dev/null @@ -1,535 +0,0 @@ -# *****************************COPYRIGHT******************************* -# (C) Crown copyright Met Office. All rights reserved. -# For further details please refer to the file COPYRIGHT.txt -# which you should have received as part of this distribution. -# *****************************COPYRIGHT******************************* -''' -## NOTE ## -This module is one of several for which the Master copy is in the -UM repository. When making changes, please ensure the changes are made in the UM -repository or they will be lost during the release process when the UM copy is -copied over. - -This module contains various functions for parsing and manuipulating -quoted strings in Fortran -''' -import re - - -class ParsingError(Exception): - ''' - Raised when an operation attempts to parse a line that doesn't look - like it expects. - ''' - def __init__(self): - self.msg = 'Parsing Error.' - pass - - -class QuoteError(ParsingError): - ''' - Raised when there are an unexpected number of quote marks (single or - double) in a line. - ''' - def __init__(self, quote, number): - self.number = number - self.quote = quote - self.quotemarks = {"'": "single quotes", - "\"": "double quotes"} - - self.msg = "There are an odd number of non-commented " \ - "and non-quoted {1:s} in this line. " \ - "(From a total of {0:d})".format(number, - self.quotemarks[quote]) - pass - - -SQUOTE = 0 -DQUOTE = 1 - -PBLANKFSTRRE = re.compile("^[^!]*?('|\")") -BLANKFCOMSRE = re.compile("('|\")(.*)!(.*)&") -ISPPCONTCR = re.compile(r"\\\s*$", flags=re.IGNORECASE) - - -def replace_characters(line, locs, lens, replchar="X"): - ''' - Replace characters in a line at particular locations. - - e.g. for the line - > line = "She said 'I like tea at Fortnum & Mason'. I prefer fish & chips." - - And locations and lengths - > locs = [32, ] - > lens = [1, ] - - The result is - > newline = replace_characters(line, locs, lens, replchar="+"): - > print newline - She said 'I like tea at Fortnum + Mason'. I prefer fish & chips. - - ''' - - # This code requires that the replacement is a single string, which is - # used as many times as necessary. - if len(replchar) != 1: - raise ValueError("Argument 'replchar' must be a single character") - - # Split the line into a list, as characters can't be changed in place in - # a string. - newline = list(line) - - # Loop through locations and lengths and replace each character with the - # replacement character character. - for loc, ln in zip(*[locs, lens]): - for l in range(ln): - newline[loc+l] = replchar - - # Return the newline, joined back together as a string. - return "".join(newline) - - -def blank_fstring(line, string_continuation=[False, False]): - "blanks out strings within the fortran line" - - # partially blank the strings - bline = partial_blank_fstring(line, string_continuation) - - # if we failed to get all the single or double quotes that aren't in a - # comment, something has gone wrong - match = PBLANKFSTRRE.search(bline) - - if match is not None: - # match character is a solo - something has gone wrong - raise QuoteError(match.group(1), len(re.findall(match.group(1), - line))) - - return bline - - -def partial_blank_fstring(line, string_continuation=[False, False]): - "blanks out strings within the fortran line" - - bline = clean_str_continuation(line, string_continuation) - - if not PBLANKFSTRRE.search(bline): - # all strings are after the start of comments; return the original line - return bline - - while 1: - finder = bline.find - - # find the first remaining ' - apos_loc = finder("'") - - # find the first remaining " - quot_loc = finder("\"") - - if (apos_loc == -1 and quot_loc == -1): - # no strings remaining - break - - if (apos_loc == -1): - apos_loc = quot_loc + 1 - if (quot_loc == -1): - quot_loc = apos_loc + 1 - - first_loc = min(apos_loc, quot_loc) - - # find the first remaining ! - bang_loc = finder("!") - - if (bang_loc != -1 and bang_loc < first_loc): - # all remaining strings are after the start of comments - break - - matchchar = line[first_loc] - - match = re.search(matchchar+'.*?'+matchchar, bline) - - if match is not None: - start = match.start() - end = match.end() - bline = replace_characters(bline, [start], [end-start], - replchar=' ') - else: - # match character is a solo - we have done as much as we can - break - - return bline - - -def blank_fcomments(line, string_continuation=[False, False]): - "blanks out comments within the fortran line" - - bline = line - - # find the quoted strings to avoid non-comment ! characters - modified_line = partial_blank_fstring(line, string_continuation) - - # mop up potential string continuation containing ! characters - while BLANKFCOMSRE.search(modified_line, re.IGNORECASE): - modified_line = BLANKFCOMSRE.sub(r"\1\2 \3&", modified_line) - - # any leftover ! must be comments - if "!" in modified_line: - start = modified_line.index("!") - end = len(bline) - bline = replace_characters(bline, [start], [end-start], replchar=' ') - - return bline - - -def is_continuation(line, string_continuation=[False, False]): - "checks if line is a continuation line" - - # Now remove any strings and comments to simplify later parsing. - # (Prevent false &, ', or " character matches from strings and comments) - parblanked = partial_blank_fstring(line, string_continuation) - parblanked = blank_fcomments(parblanked) - - # The line may have a string continuation. If it's a - # string continuation, it follows it's a regular continuation too. - str_cont_test = is_str_continuation_preparblank(parblanked, line) - if str_cont_test[SQUOTE] is True or str_cont_test[DQUOTE] is True: - return True - - cont = False - - # ignore pp lines - they can have & characters in the context - # or logical && tests - strip_modify = parblanked.strip() - if len(strip_modify) > 0: - if strip_modify[0] == "#": - return False - - if "&" in parblanked: - cont = True - - return cont - - -def is_pp_continuation(line): - "checks if line is a pre-processing continuation line" - - if ISPPCONTCR.search(line): - return True - - return False - - -def is_str_continuation_preparblank(parblanked, line): - """worker routine to check if line is a string continuation without - directly calculating the blanked line (for performance reasons) - """ - cont = [False, False] - - finder = parblanked.find - - # any & characters left over are continuations. - # If there are no continuations, there can be no string continuation - if finder("&") == -1: - return cont - - # pp lines cannot be string continuations - strip_modify = parblanked.strip() - if len(strip_modify) > 0: - if strip_modify[0] == "#": - return cont - - # find the first remaining ' - apos_loc = finder("'") - - # find the first remaining " - quot_loc = finder("\"") - - if (apos_loc == -1 and quot_loc == -1): - # no strings remaining; ergo no string continuation - return cont - - # find which of ' or " occurs first - if (apos_loc == -1): - apos_loc = quot_loc + 1 - if (quot_loc == -1): - quot_loc = apos_loc + 1 - - first_loc = min(apos_loc, quot_loc) - - # set the correct return based on the first occurance character - if line[first_loc] == "'": - cont[SQUOTE] = True - else: - cont[DQUOTE] = True - - return cont - - -def is_str_continuation(line, string_continuation=[False, False]): - "checks if line is a string continuation" - - # Now remove any strings and comments to simplify later parsing. - # (Prevent false &, ', or " character matches from strings and comments) - parblanked = partial_blank_fstring(line, string_continuation) - parblanked = blank_fcomments(parblanked) - - return is_str_continuation_preparblank(parblanked, line) - - -def clean_str_continuation(line, string_continuation=[False, False]): - "blanks out strings withing the fortran line" - - # If we are string continuation, first deal with the leading partial string - if string_continuation[SQUOTE]: - out_line = re.sub("^(.*?)'", r"\1 ", line) - elif string_continuation[DQUOTE]: - out_line = re.sub("^(.*?)\"", r"\1 ", line) - else: - out_line = line - - return out_line - - -def simplify_line(lines): - "A pre-processor for the lines to make them easier to handle" - - # Note we will be passed a slice to include the lines after the - # current line to the end of the file to allow handling of - # continutations - iline = 0 - line = lines[iline] - - repeat_simplify = False - - if is_pp_continuation(line): - repeat_simplify = True - elif is_continuation(line): - repeat_simplify = True - - # Pull any continuation lines into this line - while repeat_simplify: - - while is_pp_continuation(line): - iline += 1 - line = ''.join([re.sub(r"\\(\s*)$", r" \1", line), lines[iline]]) - - # blank any strings pulled in, in case they contain a ! character - try: - line = blank_fstring(line) - except ParsingError as e: - str_cont_test = is_str_continuation(line) - if not (str_cont_test[SQUOTE] or str_cont_test[DQUOTE]): - print("Indentation simplify line has failed. [2]") - print("{0:s} Line simplification has failed for:".format(e.msg)) - print(line) - exit(1) - - # Blank out possible trailing comments - if "!" in line: - line = blank_fcomments(line) - - # If this results in an empty line we are done - if line.strip() == "": - return line - - iline += 1 - - while 1: - xiline = iline - xline = lines[xiline] - - while is_pp_continuation(xline): - xiline += 1 - xline = ''.join([re.sub(r"\\(\s*)$", r" \1", xline), - lines[xiline]]) - - xline = clean_str_continuation(xline, is_str_continuation(line)) - - # Skip following lines if they contain only comments, - # pre-processor directives, or are empty - if (re.search(r"^\s*$", blank_fcomments(xline), - flags=re.IGNORECASE) - or re.search(r"^\s*#\w+", xline, flags=re.IGNORECASE)): - iline = xiline + 1 - else: - break - - line = ''.join([re.sub(r"&(\s*)$", r" \1", line), lines[iline]]) - - if not is_pp_continuation(line): - if not is_continuation(line): - repeat_simplify = False - - # if the line still continues in some form, we have mis-parsed - if is_continuation(line) or is_pp_continuation(line): - print("Indentation simplify line has failed. [3]") - print("Line still appears to have continuations after parsing. " \ - "Line simplification has failed for:") - print(line) - exit(1) - - # Repeat the substitution for strings (in case any were pulled - # into the string above) - try: - line = blank_fstring(line) - except ParsingError as e: - print("Indentation simplify line has failed. [3]") - print("{0:s} Line simplification has failed for:".format(e.msg)) - print(line) - exit(1) - - # Comments - blank out any comments on the trailing line too - if "!" in line: - line = blank_fcomments(line) - - # Brackets - remove any nested brackets - # (i.e. leave only top level brackets) - # this is to aid with pattern matching where brackets are included - bracket_nest_level = 0 - new_line = "" - for char in (line): - if char == "(": - bracket_nest_level += 1 - if bracket_nest_level > 1: - new_line += " " - continue - if char == ")": - if bracket_nest_level > 1: - new_line += " " - bracket_nest_level -= 1 - continue - bracket_nest_level -= 1 - new_line += char - - line = new_line - return line - - -def find_quoted_char(line, char, string_continuation=[False, False]): - ''' - Check if a particular string (char) is present inside double or single - quotes within a line of text, and return locations of any occurrences - within the line. - - e.g. for a line containing a quote, - > line = "She said 'I like tea at Fortnum & Mason'. I prefer fish & chips." - ^ - The function - - > locs = find_quoted_char(line, "&") - - will return the location of the first ampersand - - > print locs - [32,] - - as it is within the single quoted speech. The location of the second - ampersand is not returned, as it is not within a quote in the line. - - Note that this assumes that the line has been pre-processed to remove - apostrophes. It will fail if there are an odd number of single or - double quotes on the line. - ''' - - # First check there are any instances of char in this line. If not, just - # leave without doing anything. - if re.search(char, line) is None: - return None - - # Then check if there are any quote marks. If not, just leave without doing - # anything - if re.search("['\"]", line) is None: - return None - - try: - blanked_str = blank_fstring(line, string_continuation) - except QuoteError: - str_cont_test = is_str_continuation(line, string_continuation) - if not (str_cont_test[SQUOTE] or str_cont_test[DQUOTE]): - raise - blanked_str = partial_blank_fstring(line, string_continuation) - - char_loc = [] - - # If the character is quoted, it will have been removed from the - # blanked_str version. Therefore, any difference between the strings at the - # location of the character in the original is a quoted character location. - for match in re.finditer(char, line): - i = match.start() - if (line[i] != blanked_str[i]): - char_loc.append(i) - - if len(char_loc) > 0: - return char_loc - else: - return None - - -def find_commented_char(line, char, string_continuation=[False, False]): - r''' - Check if a particular string (char) is present inside a Fortran comment - within a line of text, and return locations of any occurrences within the - line. - - e.g. for a line of Fortran code, say we want to find the opening - parenthesis in the comment - > line = "INTEGER, INTENT(IN) :: land_pts ! No. land points (to run)" - ^ - The function - - > locs = find_commented_char(line, r"\(") - - will return the location of the parenthesis in the comment - - > print locs - [53,] - - as it is within the commented text. The location of the parenthesis - around IN is not returned, as it before the quote starts. - - Note that this will give unexpected results if there are bangs that aren't - comment markers. It assumes that the line has been pre-processed with - find_quoted_char to remove any bangs that are within strings. - ''' - - # First check there are any instances of char, if not just leave - # without processing the line. - if re.search(char, line) is None: - return None - - # Now check if there are comments in this line. Note that this assumes that - # any bang present is a comment indicator. If there are any others (e.g. - # within quoted text), then this may fail. It is recommended to pre-process - # the line to make sure that the first bang on the line indicates the start - # of a comment. In normal usage, run find_quoted_char before - # find_commented_char to ensure it isn't a problem. - if re.search("!", line) is None: - # If there are no bangs, we can go. - return None - - try: - blanked_line = blank_fstring(line, string_continuation) - except QuoteError: - str_cont_test = is_str_continuation(line, string_continuation) - if not (str_cont_test[SQUOTE] or str_cont_test[DQUOTE]): - raise - blanked_line = partial_blank_fstring(line, string_continuation) - - # Find comment location, but only if there is actually something after the - # bang. If not, then there is no actual comment. - # Note that this assumes that the first bang that is found is the start of - # a comment (it assumes that any within quotes have already been dealt - # with). - comment = re.search("!.+", blanked_line) - - # If there is a comment, find any instances of the char in the comment. - if comment is None: - char_loc = [] - else: - char_loc = [loc.start() + comment.start() - for loc in re.finditer(char, comment.group())] - - # Return the location of any commented characters. - if len(char_loc) > 0: - return char_loc - else: - return None diff --git a/rose-stem/bin/indentation.py b/rose-stem/bin/indentation.py deleted file mode 100755 index 85e7eac..0000000 --- a/rose-stem/bin/indentation.py +++ /dev/null @@ -1,349 +0,0 @@ -#!/usr/bin/env python3 -# -# *****************************COPYRIGHT******************************* -# (C) Crown copyright Met Office. All rights reserved. -# For further details please refer to the file COPYRIGHT.txt -# which you should have received as part of this distribution. -# *****************************COPYRIGHT******************************* -''' -## NOTE ## -This module is one of several for which the Master copy is in the -UM repository. When making changes, please ensure the changes are made in the UM -repository or they will be lost during the release process when the UM copy is -copied over. - -Module containing various functions used to apply UMDP3 style indentation -to Fortran source code -''' -import re -import sys -from fstring_parse import * - -# Number of spaces of indent to apply per indentation level -INDENT = 2 - -# Patterns which match any line which indicates the following line should -# be indented by the above amount -INDENTATION_START = [ - # DO statement - possible label prefix followed by "DO" - r"(^\s*|^\s*\w+\s*:\s*)DO(\s+|$)", - # SELECT statement - possible label prefix followed by "SELECT CASE" - r"(^\s*|^\s*\w+\s*:\s*)SELECT\s+(CASE|TYPE)(\s*|\()", - # CASE statement - r"^\s*CASE(\s*|\()", - # CLASS IS statement - r"^\s*CLASS\s*IS(\s*|\()", - # IF statement - must end in "THEN" with possible label prefix - r"^\s*(|\w+\s*:|[0-9]*)\s*IF\s*\(.*?\)\s*THEN\s*$", - # ELSE statement - as above but allow "ELSE IF" - r"^\s*ELSE\s*(IF|$)", - # TYPE definition statement - must be followed by a word but can - # contain optional arguments - r"^\s*TYPE\s*((,\s*\w+(|\s*\(\s*\w+\s*\)\s*)\s*)*\s*::|)\s*\w+\s*$", - # WHERE statement - only if missing assignment statement - r"^\s*WHERE\s*\([^\(]*?\)\s*$", - # ELSEWHERE statement - r"^\s*ELSE\s*WHERE\s*(\(.*?\)[^\w]*$|$)", - # (ABSTRACT) INTERFACE statement - possibly followed by label suffix - r"^\s*(ABSTRACT\s*)?INTERFACE" \ - r"(\s+\w+\s*$|\s*$|" \ - r"\s+ASSIGNMENT\s*\(\s*=\s*\)\s*$|" \ - r"\s+OPERATOR\s*\(.*\)\s*$|" \ - r"\s+(READ|WRITE)\s*\(\s*(UN)?FORMATTED\s*\)\s*$)", - # ENUMs - r"^\s*ENUM\s*,\s*BIND\s*\(\s*C\s*\)\s*$", -] - -# Patterns which match any line which signifies that it and subsequent -# lines should be dedented by the above amount -INDENTATION_END = [ - # END DO statement - possibly followed by label suffix - r"^\s*([0-9]*\s*)END\s*DO(\s+\w+\s*$|\s*$)", - # END SELECT statement - possibly followed by label suffix - r"^\s*END\s*SELECT(\s+\w+\s*$|\s*$)", - # CASE statement - r"^\s*CASE(\s*|\()", - # CLASS IS statement - r"^\s*CLASS\s*IS(\s*|\()", - # ELSE statement - as above, counts as beggining and end - r"^\s*ELSE\s*(IF|$)", - # END IF statement - possibly followed by label suffix - r"^\s*END\s*IF(\s+\w+\s*$|\s*$)", - # END TYPE statement - r"^\s*END\s*TYPE(\s+\w+|)\s*$", - # END WHERE statement - r"^\s*END\s*WHERE\s*$", - # ELSEWHERE statement - r"^\s*ELSE\s*WHERE\s*(\(.*?\)[^\w]*$|$)", - # END INTERFACE statement - possibly followed by label suffix - r"^\s*END\s*INTERFACE" \ - r"(\s+\w+\s*$|\s*$|" \ - r"\s+ASSIGNMENT\s*\(\s*=\s*\)\s*$|" \ - r"\s+OPERATOR\s*\(.*\)\s*$|" \ - r"\s+(READ|WRITE)\s*\(\s*(UN)?FORMATTED\s*\)\s*$)", - # END ENUM statement - r"^\s*END\s*ENUM\s*$", -] - - -def get_current_indent(line): - "Returns the white-space at the start of a given line" - return re.search(r"^(?P\s*)([^\s]|$)", line).group("space") - - -def indent_line(line, indentation): - "Returns the given line adjusted by the required amount" - indentation_str = " "*abs(indentation) - if indentation > 0: - return indentation_str + line - elif indentation < 0: - if re.search("^"+indentation_str, line): - return re.sub(indentation_str, "", line, count=1) - else: - return line.lstrip() - else: - return line - - -def apply_indentation(lines, debug=False): - "Apply the indentation rules to a list of lines" - indentation = 0 - relative_indent = -1 - continuation = False - str_continuation = [False, False] - pp_continuation = False - new_lines = [] - indent_pp_level = 0 - indent_pp_level_start = {} - rel_indent_pp_level_start = {} - indent_pp_level_max = {} - rel_indent_pp_level_max = {} - - for iline, line in enumerate(lines): - - if debug: - print("\n{0:d}: \"{1:s}\"".format(iline, line)) - - # If this line is continuing a previous preprocessing line, - # just ignore indentation - if pp_continuation: - new_lines.append(line) - # If the next line is not a continuation reset the flag - if not is_pp_continuation(line): - if debug: - print(" (End of pre-processor continuation)") - pp_continuation = False - - # Ignore line if an OMP directive - elif (re.search(r"^\s*!\$.*", line, flags=re.IGNORECASE)): - if debug: - print(" (OMP comment)") - new_lines.append(re.sub(r"^\s*!", "!", line)) - - # Ignore line if an fcm DEPENDS ON comment - elif (re.search(r"^\s*!\s*DEPENDS\s*ON\s*:", - line, flags=re.IGNORECASE)): - if debug: - print(" (DEPENDS ON comment)") - new_lines.append(re.sub(r"^\s*!", "!", line)) - - # Ignore line if a misc compiler directive - elif (re.search(r"^\s*!(DIR|DEC|HPF|GCC|PGI|FPP|MIC)\$.*", - line, flags=re.IGNORECASE)): - if debug: - print(" (Compiler Directive comment)") - new_lines.append(re.sub(r"^\s*!", "!", line)) - - # If the entire line is a comment with the comment character - # in the first position on the line, indent just the comment - # text to line up with the current indentation level - elif re.search("^!.*$", line, flags=re.IGNORECASE) and indentation > 0: - if debug: - print(" (Comment only line (from zero indent))") - - # Get the current indentation level of the line - current_indent = get_current_indent(line) - - # Calculate the relative indent (how far the line must be - # shifted by to meet the desired indentation level) - relative_indent = indentation - len(current_indent) - - new_lines.append(indent_line(line, relative_indent)) - - # If the line contains a pre-processor directive, set to - # zero indentation. - elif re.search(r"^\s*#\w+", line, flags=re.IGNORECASE): - if debug: - print(" (Pre-processor line)") - if re.search(r"^\s*#if", line, flags=re.IGNORECASE): - if debug: - print(" (Pre-processor if line)") - indent_pp_level = indent_pp_level + 1 - indent_pp_level_start[indent_pp_level] = indentation - rel_indent_pp_level_start[indent_pp_level] = relative_indent - indent_pp_level_max[indent_pp_level] = 0 - rel_indent_pp_level_max[indent_pp_level] = 0 - elif re.search(r"^\s*#el(se|if)", line, flags=re.IGNORECASE): - if debug: - print(" (Pre-processor else/elif line)") - imax = max(indentation, indent_pp_level_max[indent_pp_level]) - indent_pp_level_max[indent_pp_level] = imax - if indentation == indent_pp_level_max[indent_pp_level]: - rel_indent_pp_level_max[indent_pp_level] = relative_indent - indentation = indent_pp_level_start[indent_pp_level] - relative_indent = rel_indent_pp_level_start[indent_pp_level] - elif re.search(r"^\s*#endif", line, flags=re.IGNORECASE): - if debug: - print(" (Pre-processor endif line)") - indent_pp_level_start.pop(indent_pp_level) - rel_indent_pp_level_start.pop(indent_pp_level) - imax = max(indentation, indent_pp_level_max[indent_pp_level]) - indent_pp_level_max[indent_pp_level] = imax - if indentation == indent_pp_level_max[indent_pp_level]: - rel_indent_pp_level_max[indent_pp_level] = relative_indent - indentation = indent_pp_level_max[indent_pp_level] - relative_indent = rel_indent_pp_level_max[indent_pp_level] - indent_pp_level_max.pop(indent_pp_level) - rel_indent_pp_level_max.pop(indent_pp_level) - indent_pp_level = indent_pp_level - 1 - new_lines.append(re.sub(r"^\s*#", "#", line)) - - # If the line is entirely blank, don't mess with the settings - # for continuation or the indentation value - elif re.search(r"^\s*$", line, flags=re.IGNORECASE): - if debug: - print(" (Blank line)") - new_lines.append(line) - - # If the line is only a comment not starting in the first column - # copy the indentation of the previous line, unless that would - # push the comment to a lower indentation level than the current - # level (in which case re-indent it like a normal line of code) - elif re.search(r"^\s*!.*$", line, flags=re.IGNORECASE): - if debug: - print(" (Comment only line)") - - # Get the current indentation level of the line - current_indent = get_current_indent(line) - if debug: - print(" current indent = " + str(len(current_indent))) - print(" required indent = " + str(indentation)) - # Check to see if the current relative indent will push this - # line below the current level (and adjust the relative - # indent if necessary) - if relative_indent + len(current_indent) < indentation: - relative_indent = indentation - len(current_indent) - if debug: - print(" relative_indent = " + str(relative_indent)) - new_lines.append(indent_line(line, relative_indent)) - - # If this line is continuing a previous line, instead of - # trying to calculate its indentation just apply the same - # indentation as the previous line and move on - elif continuation: - new_lines.append(indent_line(line, relative_indent)) - # If the next line is not a continuation reset the flag - if is_continuation(line, str_continuation): - # check if still string continuation - str_continuation = is_str_continuation(line, str_continuation) - else: - if debug: - print(" (End of continuation)") - continuation = False - str_continuation = [False, False] - else: - # Generate a simplified version of the line for use in - # pattern matches - simple_line = simplify_line(lines[iline:]) - - if debug: - print("??" + " "*len("{0:d}".format(iline)) + - "\"{0:s}\"".format(simple_line)) - - # Check for ending statements first - since the indentation - # shift for the end of a block must also be applied to the - # line containing the block ending statement - for pattern in INDENTATION_END: - if re.search(pattern, simple_line, flags=re.IGNORECASE): - if debug: - print(" (End, matches {0:s})".format(pattern)) - indentation -= INDENT - - # Get the current indentation level of the line - current_indent = get_current_indent(line) - - # Calculate the relative indent (how far the line must be - # shifted by to meet the desired indentation level) - relative_indent = indentation - len(current_indent) - - # Indent the line by the required amount and save it to - # the output array - indented_line = indent_line(line, relative_indent) - - if debug: - print("=>" + " "*len("{0:d}".format(iline)) + - "\"{0:s}\"".format(indented_line)) - - new_lines.append(indented_line) - - # Now check for starting statements - these will affect - # the indentation level of future lines (but not the - # current line) - for pattern in INDENTATION_START: - if re.search(pattern, simple_line, flags=re.IGNORECASE): - if debug: - print(" (Start, matches {0:s})".format(pattern)) - indentation += INDENT - - # Finally, detect if the following line is a continuation - # of this one (and therefore requires no indentation) - if is_continuation(line): - if debug: - print(" (Next line continues)") - continuation = True - str_continuation = is_str_continuation(line) - - if debug: - if continuation: - print(" (continuation is live)") - - if pp_continuation: - print(" (pre-processing continuation is live)") - - # Check for the start of new pre-processor continuation - if is_pp_continuation(line) and not pp_continuation: - if debug: - print(" (Next line pre-processor continues)") - pp_continuation = True - - # Sanity check - indentation should be back to 0 by the end, - # if not then error - - if indent_pp_level != 0: - print("Pre-processing branch level non-zero") - return None - - if indentation != 0: - print("Final indentation level non-zero ({0:d})".format(indentation)) - return None - - return new_lines - - -def main(): - '''Main toplevel function for testing''' - input_file = sys.argv[1] - - with open(input_file, "r+") as file_in: - lines_in = file_in.read().split("\n") - new_lines = apply_indentation(lines_in, debug=len(sys.argv) > 2) - if new_lines is not None: - file_in.seek(0) - file_in.write("\n".join(new_lines)) - file_in.truncate() - else: - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/rose-stem/bin/styling.py b/rose-stem/bin/styling.py deleted file mode 100755 index 2eb9ccb..0000000 --- a/rose-stem/bin/styling.py +++ /dev/null @@ -1,1141 +0,0 @@ -#!/usr/bin/env python3 -# *****************************COPYRIGHT******************************* -# (C) Crown copyright Met Office. All rights reserved. -# For further details please refer to the file COPYRIGHT.txt -# which you should have received as part of this distribution. -# *****************************COPYRIGHT******************************* -''' -## NOTE ## -This module is one of several for which the Master copy is in the -UM repository. When making changes, please ensure the changes are made in the UM -repository or they will be lost during the release process when the UM copy is -copied over. - -This module contains various functions for applying UMDP3 styling to -Fortran source code -''' -import re -import sys -from fstring_parse import * - -CODE_REPLACEMENTS = [ - # Replace Fortran 77 style conditional keywords - (r'\.eq\.', ' == '), - (r'\.ne\.', ' /= '), - (r'\.gt\.', ' > '), - (r'\.lt\.', ' < '), - (r'\.ge\.', ' >= '), - (r'\.le\.', ' <= '), - # protect 'operator' definitions e.g. "OPERATOR(/)", from the array - # initiialisation enforcement by enforcing spaces around the operators. - # Make all operators follow the same style but only (/) and (/=) had issues. - (r'\(\s*\*\s*\)', '( * )'), - (r'\(\s*\+\s*\)', '( + )'), - (r'\(\s*-\s*\)', '( - )'), - (r'\(\s*\/\s*\)', '( / )'), - (r'\(\s*==\s*\)', '( == )'), - (r'\(\s*\/=\s*\)', '( /= )'), - (r'\(\s*<\s*\)', '( < )'), - (r'\(\s*<=\s*\)', '( <= )'), - (r'\(\s*>\s*\)', '( > )'), - (r'\(\s*>=\s*\)', '( >= )'), - # Replace array initialisations - (r'\(\/', '['), - (r'\/\)', ']'), - # Ensure remaining comparitive logicals have spaces either side - (r'([^\s])(? .not.'), - (r'\.not\.([^\s])', r'.not. \g<1>'), - (r'([^\s])\.and\.', r'\g<1> .and.'), - (r'\.and\.([^\s])', r'.and. \g<1>'), - (r'([^\s])\.or\.', r'\g<1> .or.'), - (r'\.or\.([^\s])', r'.or. \g<1>'), - (r'([^\s])\.eqv\.', r'\g<1> .eqv.'), - (r'\.eqv\.([^\s])', r'.eqv. \g<1>'), - (r'([^\s])\.neqv\.', r'\g<1> .neqv.'), - (r'\.neqv\.([^\s])', r'.neqv. \g<1>'), - # Ensure hard-coded real numbers have a zero after the decimal point - (r'([0-9])\.([^0-9]|$)', r'\g<1>.0\g<2>'), - # Remove start of line ampersands, without changing the spacing of - # the line they appear on - (r'^(\s*)&(.*\w.*)$', r'\g<1> \g<2>'), - # Make constructs which include brackets have exactly one space - # between the construct and the bracket character - (r'(^\s*)(\w+\s*:\s*|[0-9]+\s*|)((else|)\s*if)(|\s\s+)\(', - r'\g<1>\g<2>\g<3> ('), - (r'(^\s*)(\w+\s*:\s*|[0-9]+\s*|)where(|\s\s+)\(', r'\g<1>\g<2>where ('), - (r'(^\s*)case(|\s\s+)\(', r'\g<1>case ('), - (r'\)(|\s\s+)then(\W|$)', r') then\g<1>'), - # Make intent statements contain no extra spacing inside the brackets - (r'(.*intent\s*)\(\s*in\s*\)(.*)', r'\g<1>(in)\g<2>'), - (r'(.*intent\s*)\(\s*out\s*\)(.*)', r'\g<1>(out)\g<2>'), - (r'(.*intent\s*)\(\s*in out\s*\)(.*)', r'\g<1>(in out)\g<2>'), - # Make module USE, ONLY statments have exactly no space between the ONLY - # and the colon character after it - (r'^(\s*)use(\s*,\s*\w+\s*::|)(\s+\w+\s*,\s*)only\s*:(.*)$', - r'\g<1>use\g<2>\g<3>only:\g<4>'), -] - -COMMENT_REPLACEMENTS = [ - # DEPENDS ON fcm constructions - (r'^(\s*!)\s*depends\s*on\s*:\s*', r'\g<1> DEPENDS ON: '), -] - -FORTRAN_TYPES = [ - "CHARACTER", - "CLASS", - "COMPLEX", - "DOUBLE PRECISION", - "ENUMERATOR", - "INTEGER", - "LOGICAL", - "REAL", - "TYPE", -] - -KEYWORDS = set([ - "abort", - "abs", - "abstract", - "access", - "achar", - "acos", - "acosd", - "acosh", - "action", - "adjustl", - "adjustr", - "advance", - "aimag", - "aint", - "alarm", - "algama", - "all", - "allocatable", - "allocate", - "allocated", - "alog", - "alog10", - "amax0", - "amax1", - "amin0", - "amin1", - "amod", - "and", - "anint", - "any", - "asin", - "asind", - "asinh", - "assign", - "assignment", - "associate", - "associated", - "asynchronous", - "atan", - "atan2", - "atan2d", - "atand", - "atanh", - "atomic", - "atomic_add", - "atomic_and", - "atomic_cas", - "atomic_define", - "atomic_fetch_add", - "atomic_fetch_and", - "atomic_fetch_or", - "atomic_fetch_xor", - "atomic_int_kind", - "atomic_logical_kind", - "atomic_or", - "atomic_ref", - "atomic_xor", - "backspace", - "backtrace", - "barrier", - "besj0", - "besj1", - "besjn", - "bessel_j0", - "bessel_j1", - "bessel_jn", - "bessel_y0", - "bessel_y1", - "bessel_yn", - "besy0", - "besy1", - "besyn", - "bge", - "bgt", - "bind", - "bit_size", - "blank", - "ble", - "block", - "blt", - "btest", - "c_alert", - "c_associated", - "c_backspace", - "c_bool", - "c_carriage_return", - "c_char", - "c_double", - "c_double_complex", - "c_f_pointer", - "c_f_procpointer", - "c_float", - "c_float128", - "c_float128_complex", - "c_float_complex", - "c_form_feed", - "c_funloc", - "c_funptr", - "c_horizontal_tab", - "c_int", - "c_int128_t", - "c_int16_t", - "c_int32_t", - "c_int64_t", - "c_int8_t", - "c_int_fast128_t", - "c_int_fast16_t", - "c_int_fast32_t", - "c_int_fast64_t", - "c_int_fast8_t", - "c_int_least128_t", - "c_int_least16_t", - "c_int_least32_t", - "c_int_least64_t", - "c_int_least8_t", - "c_intmax_t", - "c_intptr_t", - "c_loc", - "c_long", - "c_long_double", - "c_long_double_complex", - "c_long_long", - "c_new_line", - "c_null_char", - "c_null_funptr", - "c_null_ptr", - "c_ptr", - "c_ptrdiff_t", - "c_short", - "c_signed_char", - "c_size_t", - "c_sizeof", - "c_vertical_tab", - "cabs", - "call", - "case", - "ccos", - "cdabs", - "cdcos", - "cdexp", - "cdlog", - "cdsin", - "cdsqrt", - "ceiling", - "cexp", - "char", - "character", - "character_kinds", - "character_storage_size", - "chdir", - "chmod", - "class", - "clog", - "close", - "cmplx", - "co_broadcast", - "co_max", - "co_min", - "co_reduce", - "co_sum", - "codimension", - "command_argument_count", - "common", - "compiler_options", - "compiler_version", - "complex", - "concurrent", - "conjg", - "contains", - "contiguous", - "continue", - "convert", - "copyin", - "copyprivate", - "cos", - "cosd", - "cosh", - "cotan", - "cotand", - "count", - "cpp", - "cpu_time", - "cqabs", - "cqcos", - "cqexp", - "cqlog", - "cqsin", - "cqsqrt", - "critical", - "cshift", - "csin", - "csqrt", - "ctime", - "cycle", - "dabs", - "dacos", - "dacosh", - "dasin", - "dasinh", - "data", - "datan", - "datan2", - "datanh", - "date_and_time", - "dbesj0", - "dbesj1", - "dbesjn", - "dbesy0", - "dbesy1", - "dbesyn", - "dble", - "dcmplx", - "dconjg", - "dcos", - "dcosh", - "ddim", - "deallocate", - "decode", - "default", - "deferred", - "delim", - "derf", - "derfc", - "dexp", - "dfloat", - "dgamma", - "digits", - "dim", - "dimag", - "dimension", - "dint", - "direct", - "dlgama", - "dlog", - "dlog10", - "dmax1", - "dmin1", - "dmod", - "dnint", - "do", - "dot_product", - "double", - "dprod", - "dreal", - "dshiftl", - "dshiftr", - "dsign", - "dsin", - "dsinh", - "dsqrt", - "dtan", - "dtanh", - "dtime", - "elemental", - "else", - "elsewhere", - "encode", - "end", - "endfile", - "entry", - "enum", - "enumerator", - "eor", - "eoshift", - "epsilon", - "equivalence", - "eqv", - "erf", - "erfc", - "erfc_scaled", - "errmsg", - "error", - "error_unit", - "etime", - "event_query", - "execute_command_line", - "exist", - "exit", - "exp", - "exponent", - "extends", - "extends_type_of", - "external", - "false", - "fdate", - "fget", - "fgetc", - "file", - "file_storage_size", - "final", - "firstprivate", - "float", - "floor", - "flush", - "fmt", - "fnum", - "forall", - "form", - "format", - "formatted", - "fpp", - "fput", - "fputc", - "fraction", - "free", - "fseek", - "fstat", - "ftell", - "function", - "gamma", - "generic", - "gerror", - "get_command", - "get_command_argument", - "get_environment_variable", - "getarg", - "getcwd", - "getenv", - "getgid", - "getlog", - "getpid", - "getuid", - "gmtime", - "go", - "hostnm", - "huge", - "hypot", - "iabs", - "iachar", - "iall", - "iand", - "iany", - "iargc", - "ibclr", - "ibits", - "ibset", - "ichar", - "idate", - "idim", - "idint", - "idnint", - "ieee_class", - "ieee_class_type", - "ieee_copy_sign", - "ieee_is_finite", - "ieee_is_nan", - "ieee_is_negative", - "ieee_is_normal", - "ieee_logb", - "ieee_negative_denormal", - "ieee_negative_inf", - "ieee_negative_normal", - "ieee_negative_zero", - "ieee_next_after", - "ieee_positive_denormal", - "ieee_positive_inf", - "ieee_positive_normal", - "ieee_positive_zero", - "ieee_quiet_nan", - "ieee_rem", - "ieee_rint", - "ieee_scalb", - "ieee_selected_real_kind", - "ieee_signaling_nan", - "ieee_support_datatype", - "ieee_support_denormal", - "ieee_support_divide", - "ieee_support_inf", - "ieee_support_nan", - "ieee_support_sqrt", - "ieee_support_standard", - "ieee_unordered", - "ieee_value", - "ieor", - "ierrno", - "if", - "ifix", - "imag", - "image_index", - "images", - "imagpart", - "implicit", - "import", - "in", - "include", - "index", - "inout", - "input_unit", - "inquire", - "int", - "int16", - "int2", - "int32", - "int64", - "int8", - "integer", - "integer_kinds", - "intent", - "interface", - "intrinsic", - "iomsg", - "ior", - "iostat", - "iostat_end", - "iostat_eor", - "iostat_inquire_internal_unit", - "iparity", - "iqint", - "irand", - "is", - "is_iostat_end", - "is_iostat_eor", - "isatty", - "ishft", - "ishftc", - "isign", - "isnan", - "iso_c_binding", - "iso_fortran_env", - "itime", - "kill", - "kind", - "lastprivate", - "lbound", - "lcobound", - "leadz", - "len", - "len_trim", - "lgamma", - "lge", - "lgt", - "link", - "lle", - "llt", - "lnblnk", - "loc", - "lock", - "lock_type", - "log", - "log10", - "log_gamma", - "logical", - "logical_kinds", - "long", - "lshift", - "lstat", - "ltime", - "malloc", - "maskl", - "maskr", - "master", - "matmul", - "max", - "max0", - "max1", - "maxexponent", - "maxloc", - "maxval", - "mclock", - "mclock8", - "memory", - "merge", - "merge_bits", - "min", - "min0", - "min1", - "minexponent", - "minloc", - "minval", - "mod", - "module", - "modulo", - "move_alloc", - "mvbits", - "name", - "named", - "namelist", - "nearest", - "neqv", - "new_line", - "nextrec", - "nint", - "nml", - "non_intrinsic", - "non_overridable", - "none", - "nopass", - "norm2", - "not", - "null", - "nullify", - "num_images", - "number", - "numeric_storage_size", - "only", - "open", - "opened", - "operator", - "optional", - "or", - "ordered", - "out", - "output_unit", - "pack", - "pad", - "parallel", - "parameter", - "parity", - "pass", - "perror", - "pointer", - "popcnt", - "poppar", - "position", - "precision", - "present", - "print", - "private", - "procedure", - "product", - "program", - "protected", - "public", - "pure", - "qabs", - "qacos", - "qasin", - "qatan", - "qatan2", - "qcmplx", - "qconjg", - "qcos", - "qcosh", - "qdim", - "qerf", - "qerfc", - "qexp", - "qgamma", - "qimag", - "qlgama", - "qlog", - "qlog10", - "qmax1", - "qmin1", - "qmod", - "qnint", - "qsign", - "qsin", - "qsinh", - "qsqrt", - "qtan", - "qtanh", - "radix", - "ran", - "rand", - "random_number", - "random_seed", - "range", - "rank", - "read", - "readwrite", - "real", - "real128", - "real32", - "real64", - "real_kinds", - "realpart", - "rec", - "recl", - "record", - "recursive", - "reduction", - "rename", - "repeat", - "reshape", - "result", - "return", - "rewind", - "rewrite", - "rrspacing", - "rshift", - "same_type_as", - "save", - "scale", - "scan", - "secnds", - "second", - "sections", - "select", - "selected_char_kind", - "selected_int_kind", - "selected_real_kind", - "sequence", - "sequential", - "set_exponent", - "shape", - "shared", - "shifta", - "shiftl", - "shiftr", - "short", - "sign", - "signal", - "sin", - "sind", - "sinh", - "size", - "sizeof", - "sleep", - "sngl", - "source", - "spacing", - "spread", - "sqrt", - "srand", - "stat", - "stat_failed_image", - "stat_locked", - "stat_locked_other_image", - "stat_stopped_image", - "stat_unlocked", - "status", - "stop", - "storage_size", - "structure", - "submodule", - "subroutine", - "sum", - "symlnk", - "sync", - "system", - "system_clock", - "tan", - "tand", - "tanh", - "target", - "task", - "taskwait", - "then", - "this_image", - "threadprivate", - "time", - "time8", - "tiny", - "to", - "trailz", - "transfer", - "transpose", - "trim", - "true", - "ttynam", - "type", - "ubound", - "ucobound", - "umask", - "unformatted", - "unit", - "unlink", - "unlock", - "unpack", - "use", - "value", - "verif", - "verify", - "volatile", - "wait", - "where", - "while", - "workshare", - "write", - "xor", - "zabs", - "zcos", - "zexp", - "zlog", - "zsin", - "zsqrt", - ]) - - -def replace_patterns(line, str_continuation): - '''Replace various patterns according to the styling guidelines on - the provided line, returning the result''' - - if len(line.strip()) == 0: - return line - - workline = clean_str_continuation(line, str_continuation) - - stripline = workline.strip() - - # Pre-processor lines start with #. Ignore them completely. - pre_proc = (stripline[0] == "#") - - # Lines that are completely commented start with a bang and are also - # ignored completely. - all_comment = (stripline[0] == "!") - - if pre_proc or all_comment: - return line - - # remove strings - try: - blank_line = blank_fstring(workline) - except ParsingError as e: - str_cont_test = is_str_continuation(workline) - check_quote = str_cont_test[SQUOTE] or str_cont_test[DQUOTE] - if check_quote is True: - blank_line = partial_blank_fstring(workline) - else: - print("keyword split simplify line has failed.") - print("{0:s} Line simplification has failed for:".format(e.msg)) - print(line) - exit(1) - - # remove comments - match_line = blank_fcomments(blank_line) - - # Look for existance of code replacement patterns - for pattern, replacement in CODE_REPLACEMENTS: - m = re.search(pattern, match_line, flags=re.IGNORECASE) - if m: - for n in range(len(m.groups())+1): - replacement = re.sub(r'(\\{0:s}|\\g<{0:s}>)'.format(str(n)), - line[m.start(n):m.end(n)], - replacement) - - line = "".join([line[:m.start(0)], - replacement, - line[m.end(0):]]) - - workline = clean_str_continuation(line, str_continuation) - - # remove strings - try: - blank_line = blank_fstring(workline) - except ParsingError as e: - str_cont_test = is_str_continuation(workline) - check_quote = str_cont_test[SQUOTE] or str_cont_test[DQUOTE] - if check_quote is True: - blank_line = partial_blank_fstring(workline) - else: - print("keyword split simplify line has failed.") - print("{0:s} Line simplification has failed for:" \ - "".format(e.msg)) - print(line) - exit(1) - - # remove comments - match_line = blank_fcomments(blank_line) - - return line - - -def replace_comment_patterns(line, str_continuation): - '''Replace various patterns according to the styling guidelines on - the provided line, returning the result''' - - if len(line.strip()) == 0: - return line - - workline = clean_str_continuation(line, str_continuation) - - stripline = workline.strip() - - # Pre-processor lines start with #. Ignore them completely. - pre_proc = (stripline[0] == "#") - - if pre_proc: - return line - - # remove strings - try: - match_line = blank_fstring(workline) - except ParsingError as e: - str_cont_test = is_str_continuation(workline) - check_quote = str_cont_test[SQUOTE] or str_cont_test[DQUOTE] - if check_quote is True: - match_line = partial_blank_fstring(workline) - else: - print("keyword split simplify line has failed.") - print("{0:s} Line simplification has failed for:".format(e.msg)) - print(line) - exit(1) - - # Look for existance of code replacement patterns - for pattern, replacement in COMMENT_REPLACEMENTS: - m = re.search(pattern, match_line, flags=re.IGNORECASE) - if m: - for n in range(len(m.groups())+1): - replacement = re.sub(r'(\\{0:s}|\\g<{0:s}>)'.format(str(n)), - line[m.start(n):m.end(n)], - replacement) - - line = "".join([line[:m.start(0)], - replacement, - line[m.end(0):]]) - - workline = clean_str_continuation(line, str_continuation) - - # remove strings - try: - match_line = blank_fstring(workline) - except ParsingError as e: - str_cont_test = is_str_continuation(workline) - check_quote = str_cont_test[SQUOTE] or str_cont_test[DQUOTE] - if check_quote is True: - match_line = partial_blank_fstring(workline) - else: - print("keyword split simplify line has failed.") - print("{0:s} Line simplification has failed for:" \ - "".format(e.msg)) - print(line) - exit(1) - - return line - - -def upcase_keywords(line, str_continuation): - '''Upper-case any Fortran keywords on the given line, and down-case any - all capital words which aren't keywords, returning the result''' - - workline = clean_str_continuation(line, str_continuation) - - stripline = workline.strip() - - if len(stripline) == 0 or stripline[0] == "!" or stripline[0] == "#": - return line - - nloop = len(line) - - try: - simple_line = blank_fstring(workline) - except ParsingError as e: - str_cont_test = is_str_continuation(workline) - check_quote = str_cont_test[SQUOTE] or str_cont_test[DQUOTE] - if check_quote is True: - simple_line = partial_blank_fstring(workline) - else: - print("keyword split simplify line has failed.") - print("{0:s} Line simplification has failed for:".format(e.msg)) - print(line) - exit(1) - - # remove comments - simple_line = blank_fcomments(simple_line) - - # Split the line of code into a set of words - line_words = set(re.findall(r"[\w]+", simple_line)) - - for word in line_words: - # Exclude special "__FILE__" or "__LINE__" directives - if (word.isupper() and - not re.match(r"__\w+__", word)): - recomp = re.compile(r'(^|\b){0:s}(\b|$)'.format(word)) - simple_line = recomp.sub(r'\g<1>{0:s}' - r'\g<2>'.format(word.lower()), - simple_line) - - line_words = set([word.lower() for word in line_words]) - words_to_upcase = list(line_words.intersection(KEYWORDS)) - line = list(line) - for keyword in words_to_upcase: - recomp = re.compile(r'(?i)(^|\b){0:s}(\b|$)'.format(keyword)) - simple_line = recomp.sub(r'\g<1>{0:s}\g<2>'.format(keyword.upper()), - simple_line) - - # Now add back any comments/strings - simple_line = list(simple_line) - out_line = [] - - for i in range(nloop): - if simple_line[i] == " ": - out_line.append(line[i]) - else: - out_line.append(simple_line[i]) - - # Join the line back up into a string. - out_line = "".join(out_line) - - return out_line - - -def declaration_double_colon(iline, lines, pp_line_previous, line_previous): - ''' - Attempt to add the double colon to definition lines which do not already - have it - ''' - - line = lines[iline] - - workline = re.sub(r"\\(\s*)$", r" \1", pp_line_previous) - workline = workline + re.sub(r"&(\s*)$", r" \1", line_previous) + line - - found_dec_type = None - - for declare_type in FORTRAN_TYPES: - if re.search(r"^\s*{0:s}\W".format(declare_type), - workline, flags=re.IGNORECASE): - found_dec_type = declare_type - break - - if found_dec_type is not None: - xlines = lines[iline:] - xlines[0] = workline - - # Pre-process the line to pull in any continuation lines - simple_line = simplify_line(xlines) - - if not re.search(r"\s+FUNCTION(,|\s|\()", - simple_line, flags=re.IGNORECASE): - # The presence of declaration attributes (ALLOCATABLE, - # PUBLIC, POINTER, etc) are only valid when used with - # the double-colon. Therefore after allowing for the - # presence of either a (KIND/LEN=...) statement or an - # older "*INT" declaration the first character should - # not be a comma - search = re.search( - r"^(\s*{0:s}\s*?(\(.*?\)|\*\s*[0-9]+|))\s+(\w+)".format( - found_dec_type), simple_line, flags=re.IGNORECASE) - if search: - # avoid CLASS IS, TYPE IS and CLASS DEFAULT statements - classtype = search.group(3).strip().upper() - if classtype != "IS" and classtype != "DEFAULT": - # Group 1 contains everything up to the start of the - # variable definition - endpos = search.end(1) - endpos -= len(pp_line_previous) - endpos -= len(line_previous) - if endpos > 0: - statement = line[:endpos].rstrip() - else: - statement = search.group(1).rstrip() - - # Attempt to fit the double-colon into an existing space to - # preserve indentation, otherwise just add it to the line - line = re.sub(r"^{0:s}" - r"\s\s\s\s".format(re.escape(statement)), - r"{0:s} :: ".format(statement), - line, count=1) - line = re.sub(r"^{0:s}\s*" - r"((?".format( - statement), line, count=1) - - return line - - -def apply_styling(lines): - ''' - For a lit of lines apply UMDP3 styling to each line and return - the result - ''' - - output_lines = [] - continuation = False - pp_continuation = False - str_continuation = [False, False] - pseudo_str_continuation = [False, False] - pseudo_comment = False - pp_line_previous = "" - line_previous = "" - - for iline, line in enumerate(lines): - line = declaration_double_colon(iline, lines, pp_line_previous, - line_previous) - - if pp_continuation: - if not pseudo_comment: - line = replace_patterns(line, pseudo_str_continuation) - line = replace_comment_patterns(line, pseudo_str_continuation) - line = upcase_keywords(line, pseudo_str_continuation) - else: - line = replace_patterns(line, str_continuation) - line = replace_comment_patterns(line, str_continuation) - line = upcase_keywords(line, str_continuation) - - # Check for the start of new pre-processor continuation - if is_pp_continuation(line) and not pp_continuation: - pp_continuation = True - - # test the line continuation properties of this line - if pp_continuation: - if not is_pp_continuation(line): - pp_continuation = False - pp_line_previous = "" - pseudo_comment = False - elif continuation: - if is_continuation(line, str_continuation): - # check if still string continuation - str_continuation = is_str_continuation(line, str_continuation) - else: - continuation = False - line_previous = "" - str_continuation = [False, False] - else: - # Finally, detect if the following line is a continuation - # of this one (and therefore requires no indentation) - if is_continuation(line, str_continuation): - continuation = True - str_continuation = is_str_continuation(line, str_continuation) - - # if we are a (pp) continuation, save the partial line - if pp_continuation: - pp_line_previous = ''.join([re.sub(r"\\\s*$", "", - pp_line_previous), - re.sub(r"&\s*$", "", line_previous), - line]) - line_previous = "" - pseudo_line = re.sub(r"\\\s*$", "&", pp_line_previous) - pseudo_str_continuation = is_str_continuation(pseudo_line, - str_continuation) - if not pseudo_comment: - pseudo_line = partial_blank_fstring(pseudo_line, - str_continuation) - if pseudo_line.strip()[0] == "#": - pseudo_comment = True - if pseudo_line.find("!") != -1: - pseudo_line = blank_fcomments(pseudo_line, - str_continuation) - if pseudo_line.find("!") == -1: - pseudo_comment = True - elif continuation: - line_previous = re.sub(r"&\s*$", "", line_previous) - line_previous += blank_fcomments(line, str_continuation) - - output_lines.append(line) - - return output_lines - - -def main(): - '''Main toplevel function for testing''' - input_file = sys.argv[-1] - with open(input_file, "r+") as file_in: - print("Styling "+input_file) - lines_in = file_in.read().split("\n") - new_lines = apply_styling(lines_in) - file_in.seek(0) - file_in.write("\n".join(new_lines)) - file_in.truncate() - - -if __name__ == '__main__': - main() diff --git a/rose-stem/bin/umdp3_fixer.py b/rose-stem/bin/umdp3_fixer.py deleted file mode 100755 index 1e631d7..0000000 --- a/rose-stem/bin/umdp3_fixer.py +++ /dev/null @@ -1,408 +0,0 @@ -#!/usr/bin/env python3 -# -# *****************************COPYRIGHT******************************* -# (C) Crown copyright Met Office. All rights reserved. -# For further details please refer to the file COPYRIGHT.txt -# which you should have received as part of this distribution. -# *****************************COPYRIGHT******************************* -''' -## NOTE ## -This module is one of several for which the Master copy is in the -UM repository. When making changes, please ensure the changes are made in the UM -repository or they will be lost during the release process when the UM copy is -copied over. - -Top level module for the UMDP3 fixer / code styling tool - -Usage: - To apply UMDP3 styling to a specific file or set of files: - - umdp3_fixer.py [--c_code] [ ...] - - By default files are assumed to be Fortan, unless the - --c_code flag is used. - - Or to apply UMDP3 styling to files showing differences to - the fcm controlled trunk: - - umdp3_fixer.py --branch-diff - - Fortran files must end in .f90 or .F90 extension for the - branch-diff version to apply to them. C files must have the - .c or .h extension. -''' -import os -import re -import sys -import subprocess -from argparse import ArgumentParser -from shutil import which -from indentation import apply_indentation -from styling import apply_styling -from ampersands import apply_ampersand_shift -from whitespace import apply_whitespace_fixes - - -def get_branch_diff(): - '''If in a local working copy of an FCM branch, return a - list of files returned by running a branch-diff command''' - - # Use the bdiff command to extract a list of files that have changed - # on the user's branch - bdiff = subprocess.Popen("fcm bdiff --summarize", - stdout=subprocess.PIPE, - shell=True) - - bdiff_stdout, _ = bdiff.communicate() - bdiff_stdout = bdiff_stdout.decode(sys.stdout.encoding) - - bdiff_files = None - - if bdiff.returncode == 0: - bdiff_files = bdiff_stdout.strip().split("\n") - if len(bdiff_files) == 1 and bdiff_files[0] == "": - bdiff_files = None - else: - bdiff_files = [bfile[1:].strip() for bfile in bdiff_files - if bfile[0] == "M" or bfile[0] == "A"] - - if bdiff_files is None: - raise ValueError("Unable to run fcm bdiff command") - - # Use the fcm info command to extract the root name of the repository - info = subprocess.Popen("fcm info", - stdout=subprocess.PIPE, - shell=True) - - info_out, _ = info.communicate() - info_out = info_out.decode(sys.stdout.encoding) - - path = "" - wcr_path = "" - if info.returncode == 0: - info_out = info_out.strip().split("\n") - for line in info_out: - search = re.match("^Path: (?P.*)", line) - if search: - path = search.group("url") - search = re.match("^Working Copy Root Path: (?P.*)", line) - if search: - wcr_path = search.group("url") - - if path == "" or wcr_path == "": - raise ValueError("Unable to run fcm info command") - - # Use the fcm binfo command to extract the relative root name of - # the repository - binfo = subprocess.Popen("fcm binfo", - stdout=subprocess.PIPE, - shell=True) - - binfo_out, _ = binfo.communicate() - binfo_out = binfo_out.decode(sys.stdout.encoding) - - pbranch = "" - if binfo.returncode == 0: - binfo_out = binfo_out.strip().split("\n") - for line in binfo_out: - search = re.match("^Branch Parent: (?P.*?)@", line) - if search: - pbranch = search.group("url") - - if pbranch == "": - raise ValueError("Unable to run fcm binfo command") - - url_del = re.sub(re.escape(os.path.realpath(wcr_path)), "", - os.path.realpath(path)) - - if len(url_del) > 0: - if url_del[0] == '/': - url_del = url_del[1:] - - bdiff_files = [os.path.relpath(bfile, - os.path.join(pbranch, url_del)) - for bfile in bdiff_files] - - bdiff_files = [os.path.join(os.getcwd(), bfile) - for bfile in bdiff_files] - - # remove C files - regex = re.compile(r'(.*\/include\/other\/.*\.h$)|(.*\.c$)') - bdiff_files_f = [i for i in bdiff_files if not regex.search(i.strip())] - bdiff_files_c = [i for i in bdiff_files if i not in bdiff_files_f] - - return bdiff_files_f, bdiff_files_c - - -def main(): - '''Main toplevel function''' - parser = ArgumentParser(usage=""" - %(prog)s [--branch-diff] [--c_mode] [file_1 [file_2] [file_3] ...] - - This script will attempt to apply UMDP3 conformant styling to a single or - set of source files. These are assumed to be Fortran, unless the --c_mode - flag is set, in which case the source is assumed to be C. It accepts an - unlimited number of filenames as arguments and will apply styling to each - file in turn. - - NOTE: Changes to C code utilise clang-format. Because clang-format versions - are not mutually compatible, a supported version of clang-format must be - available. Additionally, as clang-format may not converge to a consistent - layout, changes to C code are not enabled by default. To enable changes to - be made, the environmnet variable RUNCCODE=1 must be set. - - NOTE: The script will overwrite the contents of the files so it is - recommended to run only on working copies which do not have uncommitted - changes - making it easy to revert should the results be undesired. - - The optional --branch-diff flag will instead assume your current directory - is within a working copy and apply the styling only to files listed by the - \"fcm branch-diff\" command. - """) - parser.add_argument("--branch-diff", dest="bdiff", - action="store_true", help="Run on a branch diff") - parser.add_argument("--c_mode", dest="c_mode", - action="store_true", - help="Assume source file(s) are C") - parser.add_argument("--col", dest="col", - type=int, - default=80, - help="Column to put &s in") - (opts, args) = parser.parse_known_args() - - if len(sys.argv) == 1: - parser.print_help() - - amp_column = opts.col - - failed = False - - if opts.bdiff: - if len(args) > 0: - sys.exit("ERROR: Cannot specify filenames and --branch-diff") - f_files, c_files = get_branch_diff() - else: - if opts.c_mode: - f_files = [] - c_files = args - else: - f_files = args - c_files = [] - - # Style Fortran Files - if opts.bdiff or not opts.c_mode: - if len(f_files) > 0: - print("\nProcessing Fortran Files") - for input_file in f_files: - print("Processing: {0:s}".format(input_file)) - sys.stdout.flush() - if (input_file.split(".")[-1] != "F90" and - input_file.split(".")[-1] != "f90" and - input_file.split(".")[-1] != "inc"): - if input_file.split(".")[-1] == "h": - if (re.search(r'.*\/include\/other\/.*', input_file) is not - None): - - print("Input file {0:s} not a " - "Fortran include file," - " skipping".format(input_file)) - continue - else: - print("Input file {0:s} not a " - "Fortran file, skipping".format(input_file)) - continue - - with open(input_file, "r+", errors="replace") as file_in: - lines = file_in.read().split("\n") - - file_in.seek(0) - - reiterate = True - modify_lines = list(lines) - - while (reiterate) and (failed is False): - old_lines = list(modify_lines) - - amp_lines = None - amp_not_parsed = [] - - if failed is False: - amp_lines, amp_not_parsed = apply_ampersand_shift( - modify_lines, preclean=True, - col=amp_column) - - if len(amp_not_parsed) > 0: - print("Ampersand Alignment Failed for:" - " {0:s}".format(input_file)) - print("failed on lines:\n") - for i in amp_not_parsed: - print(str(i)+": \""+modify_lines[i]+"\"") - print("\n") - failed = True - - white_lines = None - - if failed is False: - white_lines = apply_whitespace_fixes(amp_lines) - - if white_lines is None: - print("Whitespace Fixes Failed for:" - " {0:s}".format(input_file)) - failed = True - - styled_lines = None - - if failed is False: - styled_lines = apply_styling(white_lines) - - if styled_lines is None: - print("Styling Failed for: {0:s}".format(input_file)) - failed = True - - indented_lines = None - - if failed is False: - indented_lines = apply_indentation(styled_lines) - - if indented_lines is None: - print("Indentation Failed for: {0:s}".format( - input_file)) - failed = True - - amp_lines = None - amp_not_parsed = [] - - if failed is False: - amp_lines, amp_not_parsed = apply_ampersand_shift( - indented_lines, col=amp_column) - - if len(amp_not_parsed) > 0: - print("Ampersand Alignment Failed for:" - " {0:s}".format(input_file)) - print("failed on lines:\n") - for i in amp_not_parsed: - print(str(i)+": \""+indented_lines[i]+"\"") - print("\n") - failed = True - - if failed is False: - modify_lines = amp_lines - - if modify_lines[:] == old_lines[:]: - reiterate = False - - if failed is False: - file_in.write("\n".join(modify_lines)) - file_in.truncate() - - if failed is True: - break - - # Style C Files - if (opts.bdiff or opts.c_mode) and os.environ.get('RUNCCODE') == "1": - # check if clang-format is available - if which('clang-format') is not None: - # interogate clang-format - clang_format = subprocess.Popen("clang-format --version", - stdout=subprocess.PIPE, - shell=True) - - clang_format_stdout, _ = clang_format.communicate() - if clang_format_stdout is not None: - clang_format_stdout = clang_format_stdout.decode( - sys.stdout.encoding) - else: - clang_format_stdout = '' - - clang_format_stdout = clang_format_stdout.strip().split("\n") - - regex = re.compile(r'.*version\s*(\d+)\..*') - for i in clang_format_stdout: - clang_format_ver = regex.match(i) - if clang_format_ver is not None: - clang_format_ver = clang_format_ver.group(1) - break - - # clang-format is not backwards compatible, so set up a version to - # config file mapping - format_config_map = {'14': 'um-clang_format-v12.cfg', - '12': 'um-clang_format-v12.cfg', - '11': 'um-clang_format-v11.cfg', - '10': 'um-clang_format-v10.cfg', - '3': 'um-clang_format-v3.cfg'} - format_command_map = {'14': "clang-format --style=file -i {0}", - '12': "clang-format --style=file -i {0}", - '11': "clang-format --style=file -i {0}", - '10': "clang-format --style=file -i {0}", - '3': "bash -c 'grep -i -F " - "\"/* clang-format off */\"" - " {0} ||" - " clang-format --style=file -i {0}'"} - - if clang_format_ver is not None: - clang_format_config = None - try: - clang_format_config = format_config_map[clang_format_ver] - except KeyError: - print("\nSkipping C files:") - print("\nThis utilises clang-format," - " but the version available (version " + - clang_format_ver + - ") is not supported.") - if clang_format_config is not None: - # symlink the clang-format config corresponding to the - # installed version onto the analysis path - format_confing_root = os.path.dirname(os.path.realpath( - __file__)) - clang_format_config = os.path.join(format_confing_root, - clang_format_config) - - clang_format_symlink_target = os.path.dirname( - os.path.commonprefix(c_files)) - clang_format_symlink_target = os.path.join( - clang_format_symlink_target, "_clang-format") - - os.symlink(clang_format_config, - clang_format_symlink_target) - - print("\nProcessing C Files") - for input_file in c_files: - print("Processing: {0}".format(input_file)) - # Use the clang-format command corresponding to the - # installed version on this system - args = format_command_map[clang_format_ver].format( - input_file) - - clang_format = subprocess.Popen(args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True) - - clang_format_stdout, clang_format_stderr = ( - clang_format.communicate()) - - if clang_format.returncode != 0: - clang_format_stdout = clang_format_stdout.decode( - sys.stdout.encoding) - clang_format_stderr = clang_format_stderr.decode( - sys.stdout.encoding) - print(clang_format_stderr) - print(clang_format_stdout) - failed = True - - os.unlink(clang_format_symlink_target) - else: - print("\nSkipping C files:") - print("\nThis utilises clang-format," - " but the version available can not be determined.") - else: - print("\nSkipping C files:") - print("\nThis utilises clang-format," - " which is not available on this system") - - if failed is True: - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/rose-stem/bin/whitespace.py b/rose-stem/bin/whitespace.py deleted file mode 100755 index 4b0649b..0000000 --- a/rose-stem/bin/whitespace.py +++ /dev/null @@ -1,275 +0,0 @@ -#!/usr/bin/env python3 - -## NOTE ## -# This module is one of several for which the Master copy is in the UM -# repository. When making changes, please ensure the changes are made in the UM -# repository or they will be lost during the release process when the UM copy -# is copied over. - -import sys -import re -import argparse -from fstring_parse import * - -# These 4 are defined globally for visibility. They are actually only used in -# main or a single subroutine but actual text is not yet defined in stone... - -conjoined_keywords = { - r"BLOCK\s*DATA": "BLOCK DATA", - r"DOUBLE\s*PRECISION": "DOUBLE PRECISION", - r"ELSE\s*IF": "ELSE IF", - r"ELSE\s*WHERE": "ELSE WHERE", - r"END\s*ASSOCIATE": "END ASSOCIATE", - r"END\s*BLOCK": "END BLOCK", - r"END\s*BLOCK\s*DATA": "END BLOCK DATA", - r"END\s*CRITICAL": "END CRITICAL", - r"END\s*DO": "END DO", - r"END\s*ENUM": "END ENUM", - r"END\s*FILE": "END FILE", - r"END\s*FORALL": "END FORALL", - r"END\s*FUNCTION": "END FUNCTION", - r"END\s*IF": "END IF", - r"END\s*INTERFACE": "END INTERFACE", - r"END\s*MODULE": "END MODULE", - r"END\s*PARALLEL": "END PARALLEL", - r"END\s*PARALLEL\s*DO": "END PARALLEL DO", - r"END\s*PROCEDURE": "END PROCEDURE", - r"END\s*PROGRAM": "END PROGRAM", - r"END\s*SELECT": "END SELECT", - r"END\s*SUBMODULE": "END SUBMODULE", - r"END\s*SUBROUTINE": "END SUBROUTINE", - r"END\s*TYPE": "END TYPE", - r"END\s*WHERE": "END WHERE", - r"GO\s*TO": "GO TO", - r"IN\s*OUT": "IN OUT", - r"PARALLEL\s*DO": "PARALLEL DO", - r"SELECT\s*CASE": "SELECT CASE", - r"SELECT\s*TYPE": "SELECT TYPE", -} - - -def split_conjoined_keyword(first_o_line, bit_to_change, new_bit, rest_o_line): - new_line = first_o_line + new_bit + rest_o_line - return new_line - - -def strip_trailing_space(line): - return re.sub(r"\s*$", "", line) - - -def keyword_split(line, str_continuation): - - if len(line.strip()) == 0: - return line - - workline = clean_str_continuation(line, str_continuation) - - stripline = workline.strip() - - # Pre-processor lines start with #. Ignore them completely. - pre_proc = (stripline[0] == "#") - - # Lines that are completely commented start with a bang and are also - # ignored completely. - all_comment = (stripline[0] == "!") - - if pre_proc or all_comment: - return line - - # remove strings - try: - blank_line = blank_fstring(workline) - except ParsingError as e: - str_cont_test = is_str_continuation(workline) - check_quote = str_cont_test[SQUOTE] or str_cont_test[DQUOTE] - if check_quote is True: - blank_line = partial_blank_fstring(workline) - else: - print("keyword split simplify line has failed.") - print("{0:s} Line simplification has failed for:".format(e.msg)) - print(line) - exit(1) - - # remove comments - match_line = blank_fcomments(blank_line) - - # Look for possible keyword missing a space... - for key, value in conjoined_keywords.items(): - searchstring = r"(^.*\W|^)(" + key + r")(\W.*$|$)" - - m = re.search(searchstring, match_line, re.IGNORECASE) - if m: - first_o_line = line[m.start(1):m.end(1)] - bit_to_change = line[m.start(2):m.end(2)] - rest_o_line = line[m.start(3):] - line = split_conjoined_keyword(first_o_line, bit_to_change, - value, rest_o_line) - - workline = clean_str_continuation(line, str_continuation) - - # remove strings - try: - blank_line = blank_fstring(workline) - except ParsingError as e: - str_cont_test = is_str_continuation(workline) - check_quote = str_cont_test[SQUOTE] or str_cont_test[DQUOTE] - if check_quote is True: - blank_line = partial_blank_fstring(workline) - else: - print("keyword split simplify line has failed.") - print("{0:s} Line simplification has failed " \ - "for:".format(e.msg)) - print(line) - exit(1) - - # remove comments - match_line = blank_fcomments(blank_line) - - return line - - -def apply_whitespace_fixes(lines, striptrailingspace=True, keywordsplit=True): - '''For a lit of lines apply UMDP3 styling to each line and return - the result''' - - output_lines = [] - continuation = False - pp_continuation = False - str_continuation = [False, False] - pseudo_str_continuation = [False, False] - pseudo_comment = False - pp_line_previous = "" - line_previous = "" - - for iline, line in enumerate(lines): - if striptrailingspace: - # Strip out trailing spaces ? - line = strip_trailing_space(line) - - if keywordsplit: - if pp_continuation: - if not pseudo_comment: - line = keyword_split(line, pseudo_str_continuation) - else: - line = keyword_split(line, str_continuation) - - # Check for the start of new pre-processor continuation - if is_pp_continuation(line) and not pp_continuation: - pp_continuation = True - - # test the line continuation properties of this line - if pp_continuation: - if not is_pp_continuation(line): - pp_continuation = False - pp_line_previous = "" - pseudo_comment = False - elif continuation: - if is_continuation(line, str_continuation): - # check if still string continuation - str_continuation = is_str_continuation(line, str_continuation) - else: - continuation = False - line_previous = "" - str_continuation = [False, False] - else: - # Finally, detect if the following line is a continuation - # of this one (and therefore requires no indentation) - if is_continuation(line, str_continuation): - continuation = True - str_continuation = is_str_continuation(line, str_continuation) - - # if we are a (pp) continuation, save the partial line - if pp_continuation: - pp_line_previous = ''.join([re.sub(r"\\\s*$", "", - pp_line_previous), - re.sub(r"&\s*$", "", line_previous), - line]) - line_previous = "" - pseudo_line = re.sub(r"\\\s*$", "&", pp_line_previous) - pseudo_str_continuation = is_str_continuation(pseudo_line, - str_continuation) - if not pseudo_comment: - pseudo_line = partial_blank_fstring(pseudo_line, - str_continuation) - if pseudo_line.strip()[0] == "#": - pseudo_comment = True - if pseudo_line.find("!") != -1: - pseudo_line = blank_fcomments(pseudo_line, - str_continuation) - if pseudo_line.find("!") == -1: - pseudo_comment = True - elif continuation: - line_previous = re.sub(r"&\s*$", "", line_previous) - line_previous += blank_fcomments(line, str_continuation) - - output_lines.append(line) - - return output_lines - - -def main(): - ''' - Main program code. - Expects there to be a single command line argument, which should be the - name of a file to process. It will abort if no argument is provided, or - the file cannot open. - In processing the file it will attempt to add a USE statement for the - module defining the KIND type variable if none exists. It will do this - at a Module level to save having to keep track of each and every - subroutine a file may contain. NOTE - if your file doesn't contain - a module - you're in trouble... - Then each lie is checked to see if there is a 'REAL' declaration and - processed if there is. - ''' - parser = argparse.ArgumentParser() - parser.add_argument("-k", "--keywordsplit", - help="Split potentially conjoined keywords" - " such as INOUT", - action="store_true") - parser.add_argument("-s", "--striptrailingspace", - help="Strip Trailing spaces from lines", - action="store_true") - parser.add_argument("filename", type=str, - help="Name of file to process") - args = parser.parse_args() - - modify = False - - if args.keywordsplit: - print("Splitting of potentially conjoined keywords turned on") - modify = True - else: - print("Splitting of potentially conjoined keywords turned off") - if args.striptrailingspace: - print("stripping trailing spaces turned on") - modify = True - else: - print("stripping trailing spaces turned off") - - if not modify: - print("No options selected that would cause any code changes.") - print("Exiting, as running would have no effect.") - exit(1) - - filename = args.filename - try: - with open(filename) as fortran_file: - raw_code = fortran_file.readlines() - except EnvironmentError: - print("Error opening file. :\n \"{0:s}\"\n".format(filename) + - "I need a valid filename on which to work....") - raise SystemExit - - print("\nLooking at file :\n {0:s}".format(filename)) - # re-open the fortran file, this time to write to it. - with open(filename, 'r+') as fortran_file: - lines_in = fortran_file.read().split("\n") - new_lines = apply_whitespace_fixes(lines_in, args.striptrailingspace, - args.keywordsplit) - fortran_file.seek(0) - fortran_file.write("\n".join(new_lines)) - fortran_file.truncate() - - -if __name__ == "__main__": - main() diff --git a/rose-stem/flow.cylc b/rose-stem/flow.cylc index eba8756..c206f9a 100644 --- a/rose-stem/flow.cylc +++ b/rose-stem/flow.cylc @@ -1,10 +1,30 @@ #!jinja2 +{% from "cylc.flow" import LOG %} +{% from "utils" import get_site %} +{% from "read_sources" import read_sources %} -{% set KILL = False %} +{% if SITE is not defined %} + {% set SITE = get_site() %} +{% endif %} +{% do LOG.info("Site: " + SITE) %} + +{% set SOURCE_DIRECTORY = ROSE_ORIG_HOST~":"~CYLC_WORKFLOW_SRC_DIR~"/.." %} +{# Include log message for suite_report #} +{% do LOG.info("UKCA SOURCE CLONE="~SOURCE_DIRECTORY) %} + +{# Read the sources of dependencies from the dependencies.yaml file #} +{% set dependencies = read_sources(SOURCE_DIRECTORY, "ukca", USE_HEADS) %} -{% if KILL %} - {{ "Error: No SITE in the rose.conf" / 0 }} +{# Read in groups to run #} +{% if group is defined %} + {% set RUN_NAMES = group %} +{% elif g is defined %} + {% set RUN_NAMES = g %} +{% else %} + {% set RUN_NAMES = ["developer"] %} {% endif %} +{% do LOG.info("Running groups: "~RUN_NAMES|join(' + ')) %} {# Dictionary to store site variables #} {% set SITE_VARS = {} %} @@ -18,8 +38,8 @@ [[events]] stall timeout = {{ SUITE_TIMEOUT }} abort on stall timeout = True - shutdown handlers = "suite_report.py" - stall handlers = "suite_report.py" + shutdown handlers = "suite_report_git.py -S $CYLC_WORKFLOW_RUN_DIR" + stall handlers = "suite_report_git.py -S $CYLC_WORKFLOW_RUN_DIR" {#- Import any Cylc task parameters #} %include 'include/parameters.rc' @@ -31,10 +51,10 @@ {#- Import any queues #} %include 'include/queues.rc' - [[dependencies]] - graph = """ -{# Hardcode export of Simsys_Scripts to always run #} -export_simsys_scripts + [[graph]] + R1 = """ +{# Hardcode extract of sources to always run #} +extract_source {#- Recursively add dependencies from RUN_NAMES, replacing groups with subgroups/tasks #} {%- set name_graphs_out = [] %} {%- set graphs_out = [] %} @@ -72,6 +92,15 @@ export FCM_VERSION={{FCM_VERSION}} mail events = retry, submission failed, submission timeout, execution timeout submission timeout = PT12H execution timeout = PT3H + [[[environment]]] + SOURCE_DIRECTORY = $CYLC_WORKFLOW_SHARE_DIR/source +{% for dependency, values in dependencies.items() %} + {% set str = values["source"] %} + {% if values["ref"] %} + {% set str = str~"~"~values["ref"] %} + {% endif %} + {{dependency|upper}}_SOURCE_CODE = {{str}} +{% endfor %} {#- Import family and job definitions #} %include 'include/runtime.rc' diff --git a/rose-stem/include/meto/graph.rc b/rose-stem/include/meto/graph.rc index a9985f7..263c71a 100644 --- a/rose-stem/include/meto/graph.rc +++ b/rose-stem/include/meto/graph.rc @@ -3,6 +3,7 @@ ############################################################################### {% do SITE_VARS.update({"LAUNCH_PLATFORM": "azspice"}) %} +{% do SITE_VARS.update({"GIT_MIRROR_LOC": "/data/users/gitassist/git_mirrors"}) %} ############################################################################### ## Imports @@ -13,7 +14,7 @@ ## Name graphs ############################################################################### {%- set name_graphs = { - "umdp3_check" : "umdp3_checker", + "umdp3_check" : "extract_source => umdp3_checker", } %} diff --git a/rose-stem/include/meto/queues.rc b/rose-stem/include/meto/queues.rc index 37f4481..a31d8b5 100644 --- a/rose-stem/include/meto/queues.rc +++ b/rose-stem/include/meto/queues.rc @@ -2,7 +2,3 @@ # In an attempt to limit the impact of rose stem on MONSooN we therefore limit # the number of script tasks. - [[queues]] - [[[linux_build]]] - limit = 2 - members = METO_LINUX_BUILD diff --git a/rose-stem/include/meto/runtime.rc b/rose-stem/include/meto/runtime.rc index f983182..14e2bc0 100644 --- a/rose-stem/include/meto/runtime.rc +++ b/rose-stem/include/meto/runtime.rc @@ -9,6 +9,12 @@ execution time limit = PT2M + [[extract_source]] + inherit = SOURCE_EXTRACTION + platform = {{ROSE_ORIG_HOST}} + execution time limit = PT15M + + # Redefine site-independent SCRIPTS group # (normally found in runtime-common.rc, Done to enforce loading of # scitools when running on MONSooN ; Python 3 is req'd by UMDP3 checker) diff --git a/rose-stem/include/runtime.rc b/rose-stem/include/runtime.rc index 904351c..8a00688 100644 --- a/rose-stem/include/runtime.rc +++ b/rose-stem/include/runtime.rc @@ -6,14 +6,43 @@ [[LINUX]] platform = localhost + [[SOURCE_EXTRACTION]] + post-script = """ +cp $SOURCE_DIRECTORY/ukca/dependencies.yaml $CYLC_WORKFLOW_RUN_DIR +cp $SOURCE_DIRECTORY/SimSys_Scripts/github_scripts/suite_report_git.py $CYLC_WORKFLOW_RUN_DIR/bin +cp $SOURCE_DIRECTORY/SimSys_Scripts/github_scripts/suite_data.py $CYLC_WORKFLOW_RUN_DIR/bin +cp $SOURCE_DIRECTORY/SimSys_Scripts/github_scripts/git_bdiff.py $CYLC_WORKFLOW_RUN_DIR/bin +cp $SOURCE_DIRECTORY/SimSys_Scripts/github_scripts/get_git_sources.py $CYLC_WORKFLOW_RUN_DIR/bin +""" + [[[environment]]] + ROSE_TASK_APP = extract_source + DEPENDENCIES = {{dependencies}} + USE_MIRRORS = {{USE_MIRRORS}} + USE_TOKENS = {{USE_TOKENS}} + {% if USE_MIRRORS %} + {% if "GIT_MIRROR_LOC" in SITE_VARS %} + GIT_MIRROR_LOC = {{SITE_VARS.GIT_MIRROR_LOC}} + {% else %} + {{ raise( + "Trying to run with mirrors without setting a mirror location. " + "Ensure 'GIT_MIRROR_LOC' is included in SITE_VARS." + ) }} + {% endif %} + {% endif %} + + [[SOURCE_SYNC]] + script = """ +hostname=$(rose host-select $host 2>/dev/null || echo $host) +echo "rsyncing source to $hostname" +RELATIVE_SOURCE_DIRECTORY=`echo $SOURCE_DIRECTORY | sed "s|$HOME/||"` +ssh $hostname mkdir -p $RELATIVE_SOURCE_DIRECTORY +rsync -avz --exclude=".*" $SOURCE_DIRECTORY/* $hostname:$RELATIVE_SOURCE_DIRECTORY/ +""" + # Family for extract jobs [[EXTRACT]] script = "rose task-run --verbose --define='args=--ignore-lock'" [[[environment]]] - HOST_SOURCE_UKCA_BASE = {{ HOST_SOURCE_UKCA_BASE }} -{%- if SITE == 'vm' %} - HOST_SOURCE_UKCA_BASE = {{ HOST_SOURCE_UKCA_BASE|replace(ROSE_ORIG_HOST+":","",1) }} -{%- endif %} # Family for housekeeping jobs # Accepts the environment variable DIR1, the directory to delete diff --git a/rose-stem/include/vm/graph.rc b/rose-stem/include/vm/graph.rc index df0365f..c6bc91e 100644 --- a/rose-stem/include/vm/graph.rc +++ b/rose-stem/include/vm/graph.rc @@ -7,7 +7,7 @@ ## Name graphs ############################################################################### {%- set name_graphs = { - "umdp3_check" : "umdp3_checker", + "umdp3_check" : "extract_source => umdp3_checker", } %} diff --git a/rose-stem/include/vm/runtime.rc b/rose-stem/include/vm/runtime.rc index fdbdcf6..caec87f 100644 --- a/rose-stem/include/vm/runtime.rc +++ b/rose-stem/include/vm/runtime.rc @@ -4,3 +4,8 @@ [[SCRIPTS]] inherit = None, LINUX execution time limit = PT20M + + [[extract_source]] + inherit = SOURCE_EXTRACTION + platform = localhost + execution time limit = PT15M diff --git a/rose-stem/lib/python/read_sources.py b/rose-stem/lib/python/read_sources.py new file mode 100644 index 0000000..a651ea7 --- /dev/null +++ b/rose-stem/lib/python/read_sources.py @@ -0,0 +1,60 @@ +import yaml +import tempfile +import os +from subprocess import run +from shutil import rmtree + +def get_dependencies_file(wc_loc): + """ + Copy the dependencies file to a temporary directory on the local machine that can be + read. + """ + + tempdir = tempfile.mkdtemp() + + try: + host, path = wc_loc.split(":") + path = os.path.join(path, "dependencies.yaml") + copy_command = f"scp -o StrictHostKeyChecking=no {host}:" + except ValueError: + path = os.path.join(wc_loc, "dependencies.yaml") + copy_command = "cp " + copy_command += f"{path} {tempdir}" + + result = run( + copy_command.split(), capture_output=True, text=True, timeout=120 + ) + + # Raise an error if the returncode is positive + if result.returncode: + raise RuntimeError( + f"An error occured while running the command '{copy_command}' " + "in order to read the dependencies file. The error message is:\n\n" + f"'{result.stderr}'" + ) + + return tempdir + +def read_sources(clone_source, repo, use_heads): + """ + Load the dependencies.yaml file as a dictionary + """ + + dependencies_file = get_dependencies_file(clone_source) + + with open(os.path.join(dependencies_file, "dependencies.yaml")) as stream: + dependencies = yaml.safe_load(stream) + + if not dependencies[repo]["source"]: + dependencies[repo]["source"] = clone_source + + # Populate parent, assume MetOffice is owner if not set + for dependency, values in dependencies.items(): + if "parent" not in values: + dependencies[dependency]["parent"] = f"MetOffice/{dependency}.git" + if use_heads: + dependencies[dependency]["ref"] = "" + + rmtree(dependencies_file) + + return dependencies diff --git a/rose-stem/lib/python/utils.py b/rose-stem/lib/python/utils.py new file mode 100644 index 0000000..8aa4b11 --- /dev/null +++ b/rose-stem/lib/python/utils.py @@ -0,0 +1,11 @@ +from subprocess import Popen, PIPE + + +def get_site(): + proc = Popen( + ["rose", "config", "rose-stem", "automatic-options"], stdout=PIPE, text=True + ) + out, _ = proc.communicate() + if proc.returncode or "SITE" not in out: + raise Exception('Could not determine the rose-stem "SITE"') + return out.replace("SITE=", "").strip() diff --git a/rose-stem/rose-suite.conf b/rose-stem/rose-suite.conf index f329513..60f4b56 100644 --- a/rose-stem/rose-suite.conf +++ b/rose-stem/rose-suite.conf @@ -1,5 +1,3 @@ -ROSE_STEM_VERSION=1 - [template variables] COMPARE_OUTPUT=true COMPARE_WALLCLOCK=true @@ -8,4 +6,6 @@ HOUSEKEEPING=false SIMSYS_BRANCH="main" SUITE_TIMEOUT='PT15M' RUN_NAMES=[] - +USE_HEADS=false +USE_MIRRORS=false +USE_TOKENS=false From d0fe6eeb09d38828093ca2f339252e3a823ce749 Mon Sep 17 00:00:00 2001 From: James Bruten <109733895+james-bruten-mo@users.noreply.github.com> Date: Fri, 21 Nov 2025 10:11:03 +0000 Subject: [PATCH 2/9] rename rc to cylc (#2) --- rose-stem/flow.cylc | 12 ++++++------ rose-stem/include/{graph.rc => graph.cylc} | 4 ++-- rose-stem/include/meto/{graph.rc => graph.cylc} | 0 rose-stem/include/meto/{queues.rc => queues.cylc} | 0 rose-stem/include/meto/{runtime.rc => runtime.cylc} | 2 +- .../include/{parameters.rc => parameters.cylc} | 2 +- rose-stem/include/{queues.rc => queues.cylc} | 4 ++-- rose-stem/include/{runtime.rc => runtime.cylc} | 13 +++++++------ rose-stem/include/{variables.rc => variables.cylc} | 0 rose-stem/include/vm/{graph.rc => graph.cylc} | 0 rose-stem/include/vm/{queues.rc => queues.cylc} | 0 rose-stem/include/vm/{runtime.rc => runtime.cylc} | 2 +- rose-stem/lib/python/read_sources.py | 8 +++++++- rose-stem/lib/python/utils.py | 11 ++++++++++- .../{runtime-common.rc => runtime-common.cylc} | 0 15 files changed, 37 insertions(+), 21 deletions(-) rename rose-stem/include/{graph.rc => graph.cylc} (82%) rename rose-stem/include/meto/{graph.rc => graph.cylc} (100%) rename rose-stem/include/meto/{queues.rc => queues.cylc} (100%) rename rose-stem/include/meto/{runtime.rc => runtime.cylc} (90%) rename rose-stem/include/{parameters.rc => parameters.cylc} (65%) rename rose-stem/include/{queues.rc => queues.cylc} (56%) rename rose-stem/include/{runtime.rc => runtime.cylc} (90%) rename rose-stem/include/{variables.rc => variables.cylc} (100%) rename rose-stem/include/vm/{graph.rc => graph.cylc} (100%) rename rose-stem/include/vm/{queues.rc => queues.cylc} (100%) rename rose-stem/include/vm/{runtime.rc => runtime.cylc} (87%) rename rose-stem/{runtime-common.rc => runtime-common.cylc} (100%) diff --git a/rose-stem/flow.cylc b/rose-stem/flow.cylc index c206f9a..f968aa7 100644 --- a/rose-stem/flow.cylc +++ b/rose-stem/flow.cylc @@ -30,7 +30,7 @@ {% set SITE_VARS = {} %} {#- Import variable definitions #} -%include 'include/variables.rc' +%include 'include/variables.cylc' [scheduler] UTC mode = True @@ -42,14 +42,14 @@ stall handlers = "suite_report_git.py -S $CYLC_WORKFLOW_RUN_DIR" {#- Import any Cylc task parameters #} -%include 'include/parameters.rc' +%include 'include/parameters.cylc' {#- Import the dependency graphs for the available jobs and groups #} -%include 'include/graph.rc' +%include 'include/graph.cylc' [scheduling] {#- Import any queues #} -%include 'include/queues.rc' +%include 'include/queues.cylc' [[graph]] R1 = """ @@ -103,5 +103,5 @@ export FCM_VERSION={{FCM_VERSION}} {% endfor %} {#- Import family and job definitions #} -%include 'include/runtime.rc' -%include 'runtime-common.rc' +%include 'include/runtime.cylc' +%include 'runtime-common.cylc' diff --git a/rose-stem/include/graph.rc b/rose-stem/include/graph.cylc similarity index 82% rename from rose-stem/include/graph.rc rename to rose-stem/include/graph.cylc index a8c18e9..977be57 100644 --- a/rose-stem/include/graph.rc +++ b/rose-stem/include/graph.cylc @@ -5,7 +5,7 @@ ############################################################################### {% if SITE == 'meto' %} -%include 'include/meto/graph.rc' +%include 'include/meto/graph.cylc' {% elif SITE == 'vm' %} -%include 'include/vm/graph.rc' +%include 'include/vm/graph.cylc' {% endif %} diff --git a/rose-stem/include/meto/graph.rc b/rose-stem/include/meto/graph.cylc similarity index 100% rename from rose-stem/include/meto/graph.rc rename to rose-stem/include/meto/graph.cylc diff --git a/rose-stem/include/meto/queues.rc b/rose-stem/include/meto/queues.cylc similarity index 100% rename from rose-stem/include/meto/queues.rc rename to rose-stem/include/meto/queues.cylc diff --git a/rose-stem/include/meto/runtime.rc b/rose-stem/include/meto/runtime.cylc similarity index 90% rename from rose-stem/include/meto/runtime.rc rename to rose-stem/include/meto/runtime.cylc index 14e2bc0..662cc97 100644 --- a/rose-stem/include/meto/runtime.rc +++ b/rose-stem/include/meto/runtime.cylc @@ -16,7 +16,7 @@ # Redefine site-independent SCRIPTS group -# (normally found in runtime-common.rc, Done to enforce loading of +# (normally found in runtime-common.cylc, Done to enforce loading of # scitools when running on MONSooN ; Python 3 is req'd by UMDP3 checker) [[SCRIPTS]] inherit = None, METO_AZSPICE diff --git a/rose-stem/include/parameters.rc b/rose-stem/include/parameters.cylc similarity index 65% rename from rose-stem/include/parameters.rc rename to rose-stem/include/parameters.cylc index a5a0a31..acaf661 100644 --- a/rose-stem/include/parameters.rc +++ b/rose-stem/include/parameters.cylc @@ -1,5 +1,5 @@ # Import site-specific parameter sets # {% if SITE == 'niwa' %} -# %include 'include/niwa/parameters.rc' +# %include 'include/niwa/parameters.cylc' # {% endif %} \ No newline at end of file diff --git a/rose-stem/include/queues.rc b/rose-stem/include/queues.cylc similarity index 56% rename from rose-stem/include/queues.rc rename to rose-stem/include/queues.cylc index 21c00ba..39cba7a 100644 --- a/rose-stem/include/queues.rc +++ b/rose-stem/include/queues.cylc @@ -1,7 +1,7 @@ # Import site-specific queues {% if SITE == 'meto' %} -%include 'include/meto/queues.rc' +%include 'include/meto/queues.cylc' {% elif SITE == 'vm' %} -%include 'include/vm/queues.rc' +%include 'include/vm/queues.cylc' {% endif %} diff --git a/rose-stem/include/runtime.rc b/rose-stem/include/runtime.cylc similarity index 90% rename from rose-stem/include/runtime.rc rename to rose-stem/include/runtime.cylc index 8a00688..2cca5fb 100644 --- a/rose-stem/include/runtime.rc +++ b/rose-stem/include/runtime.cylc @@ -8,11 +8,12 @@ [[SOURCE_EXTRACTION]] post-script = """ +mkdir $CYLC_WORKFLOW_RUN_DIR/bin/ cp $SOURCE_DIRECTORY/ukca/dependencies.yaml $CYLC_WORKFLOW_RUN_DIR -cp $SOURCE_DIRECTORY/SimSys_Scripts/github_scripts/suite_report_git.py $CYLC_WORKFLOW_RUN_DIR/bin -cp $SOURCE_DIRECTORY/SimSys_Scripts/github_scripts/suite_data.py $CYLC_WORKFLOW_RUN_DIR/bin -cp $SOURCE_DIRECTORY/SimSys_Scripts/github_scripts/git_bdiff.py $CYLC_WORKFLOW_RUN_DIR/bin -cp $SOURCE_DIRECTORY/SimSys_Scripts/github_scripts/get_git_sources.py $CYLC_WORKFLOW_RUN_DIR/bin +cp $SOURCE_DIRECTORY/SimSys_Scripts/github_scripts/suite_report_git.py $CYLC_WORKFLOW_RUN_DIR/bin/ +cp $SOURCE_DIRECTORY/SimSys_Scripts/github_scripts/suite_data.py $CYLC_WORKFLOW_RUN_DIR/bin/ +cp $SOURCE_DIRECTORY/SimSys_Scripts/github_scripts/git_bdiff.py $CYLC_WORKFLOW_RUN_DIR/bin/ +cp $SOURCE_DIRECTORY/SimSys_Scripts/github_scripts/get_git_sources.py $CYLC_WORKFLOW_RUN_DIR/bin/ """ [[[environment]]] ROSE_TASK_APP = extract_source @@ -58,7 +59,7 @@ rsync -avz --exclude=".*" $SOURCE_DIRECTORY/* $hostname:$RELATIVE_SOURCE_DIRECTO ## Site-specific runtime definitions ############################################################################### {% if SITE == 'meto' %} -%include 'include/meto/runtime.rc' +%include 'include/meto/runtime.cylc' {% elif SITE == 'vm' %} -%include 'include/vm/runtime.rc' +%include 'include/vm/runtime.cylc' {% endif %} diff --git a/rose-stem/include/variables.rc b/rose-stem/include/variables.cylc similarity index 100% rename from rose-stem/include/variables.rc rename to rose-stem/include/variables.cylc diff --git a/rose-stem/include/vm/graph.rc b/rose-stem/include/vm/graph.cylc similarity index 100% rename from rose-stem/include/vm/graph.rc rename to rose-stem/include/vm/graph.cylc diff --git a/rose-stem/include/vm/queues.rc b/rose-stem/include/vm/queues.cylc similarity index 100% rename from rose-stem/include/vm/queues.rc rename to rose-stem/include/vm/queues.cylc diff --git a/rose-stem/include/vm/runtime.rc b/rose-stem/include/vm/runtime.cylc similarity index 87% rename from rose-stem/include/vm/runtime.rc rename to rose-stem/include/vm/runtime.cylc index caec87f..cc849c4 100644 --- a/rose-stem/include/vm/runtime.rc +++ b/rose-stem/include/vm/runtime.cylc @@ -1,5 +1,5 @@ # Redefine site-independent SCRIPTS group -# (normally found in runtime-common.rc) +# (normally found in runtime-common.cylc) # Need to use python3 for this script [[SCRIPTS]] inherit = None, LINUX diff --git a/rose-stem/lib/python/read_sources.py b/rose-stem/lib/python/read_sources.py index a651ea7..5028716 100644 --- a/rose-stem/lib/python/read_sources.py +++ b/rose-stem/lib/python/read_sources.py @@ -1,3 +1,9 @@ +############################################################################## +# (c) Crown copyright 2025 Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## + import yaml import tempfile import os @@ -13,7 +19,7 @@ def get_dependencies_file(wc_loc): tempdir = tempfile.mkdtemp() try: - host, path = wc_loc.split(":") + host, path = wc_loc.split(":", 1) path = os.path.join(path, "dependencies.yaml") copy_command = f"scp -o StrictHostKeyChecking=no {host}:" except ValueError: diff --git a/rose-stem/lib/python/utils.py b/rose-stem/lib/python/utils.py index 8aa4b11..2f8c3d6 100644 --- a/rose-stem/lib/python/utils.py +++ b/rose-stem/lib/python/utils.py @@ -1,3 +1,9 @@ +############################################################################## +# (c) Crown copyright 2025 Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## + from subprocess import Popen, PIPE @@ -8,4 +14,7 @@ def get_site(): out, _ = proc.communicate() if proc.returncode or "SITE" not in out: raise Exception('Could not determine the rose-stem "SITE"') - return out.replace("SITE=", "").strip() + # At some sites there may be many variables that are returned by rose config rose-stem + # Try to just grab the thing after SITE= and then ignore anything afterwards + site = out.split("SITE=")[1].split(' ')[0].strip() + return site diff --git a/rose-stem/runtime-common.rc b/rose-stem/runtime-common.cylc similarity index 100% rename from rose-stem/runtime-common.rc rename to rose-stem/runtime-common.cylc From 9b69357c708c053d2b1f27260fa510e096546c4d Mon Sep 17 00:00:00 2001 From: James Bruten <109733895+james-bruten-mo@users.noreply.github.com> Date: Mon, 1 Dec 2025 11:42:38 +0000 Subject: [PATCH 3/9] Use python3 (#6) --- rose-stem/app/extract_source/rose-app.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rose-stem/app/extract_source/rose-app.conf b/rose-stem/app/extract_source/rose-app.conf index 90364b5..811e8f1 100644 --- a/rose-stem/app/extract_source/rose-app.conf +++ b/rose-stem/app/extract_source/rose-app.conf @@ -1,5 +1,5 @@ [command] -default=python rose_stem_extract_source.py +default=python3 rose_stem_extract_source.py [file:get_git_sources.py] source=git:https://github.com/MetOffice/SimSys_Scripts.git::github_scripts/get_git_sources.py::main From 96fcb7ed5ded93241b7195388244b09f8307ea74 Mon Sep 17 00:00:00 2001 From: James Bruten <109733895+james-bruten-mo@users.noreply.github.com> Date: Wed, 3 Dec 2025 10:47:05 +0000 Subject: [PATCH 4/9] add initial files (#3) Co-authored-by: Yaswant Pradhan <2984440+yaswant@users.noreply.github.com> --- .github/pull_request_template.md | 98 ++++++++++++++++++++++++ .github/workflows/check-cr-approved.yaml | 11 +++ .github/workflows/umdp3_fixer.yaml | 10 +++ .gitignore | 5 ++ LICENCE | 28 +++++++ LICENCE.txt | 12 --- README.md | 8 ++ 7 files changed, 160 insertions(+), 12 deletions(-) create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/check-cr-approved.yaml create mode 100644 .github/workflows/umdp3_fixer.yaml create mode 100644 .gitignore create mode 100644 LICENCE delete mode 100644 LICENCE.txt create mode 100644 README.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..9661b25 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,98 @@ +# PR Summary + +Sci/Tech Reviewer: +Code Reviewer: + + + + + + + + + +## Code Quality Checklist + +(_Some checks are automatically carried out via the CI pipeline_) + +- [ ] I have performed a self-review of my own code +- [ ] My code follows the project's style guidelines +- [ ] Comments have been included that aid undertanding and enhance the + readability of the code +- [ ] My changes generate no new warnings + +## Testing + +- [ ] I have tested this change locally, using the UKCA rose-stem suite +- [ ] If shared files have been modified, I have run the UM and LFRic Apps rose + stem suites +- [ ] If any tests fail (rose-stem or CI) the reason is understood and + acceptable (eg. kgo changes) +- [ ] I have added tests to cover new functionality as appropriate (eg. system + tests, unit tests, etc.) + + + +### trac.log + + + +## Security Considerations + +- [ ] I have reviewed my changes for potential security issues +- [ ] Sensitive data is properly handled (if applicable) +- [ ] Authentication and authorisation are properly implemented (if applicable) + +## Performance Impact + +- [ ] Performance of the code has been considered and, if applicable, suitable + performance measurements have been conducted + +## AI Assistance and Attribution + +- [ ] Some of the content of this change has been produced with the assistance + of _Generative AI tool name_ (e.g., Met Office Github Copilot Enterprise, + Github Copilot Personal, ChatGPT GPT-4, etc) and I have followed the + [Simulation Systems AI policy](https://metoffice.github.io/simulation-systems/FurtherDetails/ai.html) + (including attribution labels) + + + +## Documentation + +- [ ] Where appropriate I have updated documentation related to this change and + confirmed that it builds correctly + +# Sci/Tech Review + + + + +- [ ] I understand this area of code and the changes being added +- [ ] The proposed changes correspond to the pull request description +- [ ] Documentation is sufficient (do documentation papers need updating) +- [ ] Sufficient testing has been completed + +_Please alert the code reviewer via a tag when you have approved the SR_ + +# Code Review + + + +- [ ] All dependencies have been resolved +- [ ] Related Issues are properly linked and addressed +- [ ] CLA compliance is confirmed +- [ ] Code quality standards are met +- [ ] Tests are adequate and passing +- [ ] Documentation is complete and accurate +- [ ] Security considerations have been addressed +- [ ] Performance impact is acceptable diff --git a/.github/workflows/check-cr-approved.yaml b/.github/workflows/check-cr-approved.yaml new file mode 100644 index 0000000..9b66712 --- /dev/null +++ b/.github/workflows/check-cr-approved.yaml @@ -0,0 +1,11 @@ +name: Check CR approved + +on: + pull_request_review: + types: [submitted, edited, dismissed] + workflow_dispatch: + +jobs: + check_cr_approved: + if: ${{ github.event.pull_request.number }} + uses: MetOffice/growss/.github/workflows/check-cr-approved.yaml@main diff --git a/.github/workflows/umdp3_fixer.yaml b/.github/workflows/umdp3_fixer.yaml new file mode 100644 index 0000000..6f265be --- /dev/null +++ b/.github/workflows/umdp3_fixer.yaml @@ -0,0 +1,10 @@ +name: umdp3 Fixer + +on: + pull_request: + types: [opened, synchronize, reopened] + workflow_dispatch: + +jobs: + umdp3_fixer: + uses: MetOffice/growss/.github/workflows/umdp3_fixer.yaml@main diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..58d9166 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__ +.vscode +.idea +*.pyc +.pytest_cache diff --git a/LICENCE b/LICENCE new file mode 100644 index 0000000..5f36cd7 --- /dev/null +++ b/LICENCE @@ -0,0 +1,28 @@ +BSD 3-Clause Licence + +Crown Copyright (c) Met Office + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/LICENCE.txt b/LICENCE.txt deleted file mode 100644 index fee7afc..0000000 --- a/LICENCE.txt +++ /dev/null @@ -1,12 +0,0 @@ -(C) Crown copyright 2022, Met Office - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/README.md b/README.md new file mode 100644 index 0000000..5100ed2 --- /dev/null +++ b/README.md @@ -0,0 +1,8 @@ +# UKCA + +UKCA (United Kingdom Chemistry and Aerosols Project) is a joint NCAS-Met Office +programme part of the NERC-Met Office Joint Climate and Weather Research +Programme (JWCRP). Project partners are the Hadley Centre and the Universities +of Cambridge, Leeds and Oxford. Our objective is to develop, evaluate and make +available a UK community atmospheric chemistry-aerosol model suitable for a +range of topics in climate and environmental change research. From 74c5c6a19c297764602ea88f2ca151ee3e1a95d9 Mon Sep 17 00:00:00 2001 From: James Bruten <109733895+james-bruten-mo@users.noreply.github.com> Date: Wed, 3 Dec 2025 14:17:06 +0000 Subject: [PATCH 5/9] Update grammar of PR template (#7) Grammatical changes pulled in from another review --- .github/pull_request_template.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 9661b25..9b7ff30 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -89,10 +89,10 @@ _Please alert the code reviewer via a tag when you have approved the SR_ - [ ] All dependencies have been resolved -- [ ] Related Issues are properly linked and addressed -- [ ] CLA compliance is confirmed -- [ ] Code quality standards are met -- [ ] Tests are adequate and passing +- [ ] Related Issues have been properly linked and addressed +- [ ] CLA compliance has been confirmed +- [ ] Code quality standards have been met +- [ ] Tests are adequate and have passed - [ ] Documentation is complete and accurate - [ ] Security considerations have been addressed - [ ] Performance impact is acceptable From 798a3725c3a851783d8dce8cf23a84c4eed0aa87 Mon Sep 17 00:00:00 2001 From: James Bruten <109733895+james-bruten-mo@users.noreply.github.com> Date: Tue, 9 Dec 2025 13:27:29 +0000 Subject: [PATCH 6/9] add cla files (#8) --- .github/workflows/cla-check.yaml | 12 +++++++++++ CONTRIBUTORS.md | 5 +++++ README.md | 35 ++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 .github/workflows/cla-check.yaml create mode 100644 CONTRIBUTORS.md diff --git a/.github/workflows/cla-check.yaml b/.github/workflows/cla-check.yaml new file mode 100644 index 0000000..cc76b84 --- /dev/null +++ b/.github/workflows/cla-check.yaml @@ -0,0 +1,12 @@ +name: CLA Check + +on: + pull_request_target: + +jobs: + cla_check: + uses: MetOffice/growss/.github/workflows/cla-check.yaml@main + + # Optional + with: + runner: 'ubuntu-24.04' diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md new file mode 100644 index 0000000..c34eb45 --- /dev/null +++ b/CONTRIBUTORS.md @@ -0,0 +1,5 @@ +# Contributors + +| GitHub user | Real Name | Affiliation | Date | +| ----------- | --------- | ----------- | ---- | +| james-bruten-mo | James Bruten | Met Office | 2025-12-09 | diff --git a/README.md b/README.md index 5100ed2..ffe062d 100644 --- a/README.md +++ b/README.md @@ -6,3 +6,38 @@ Programme (JWCRP). Project partners are the Hadley Centre and the Universities of Cambridge, Leeds and Oxford. Our objective is to develop, evaluate and make available a UK community atmospheric chemistry-aerosol model suitable for a range of topics in climate and environmental change research. + +## Contributing Guidelines + +Welcome! + +The following links are here to help set clear expectations for everyone +contributing to this project. By working together under a shared understanding, +we can continuously improve the project while creating a friendly, inclusive +space for all contributors. + +### Contributors Licence Agreement + +Please see the +[Momentum Contributors Licence Agreement](https://github.com/MetOffice/Momentum/blob/main/CLA.md) + +Agreement of the CLA can be shown by adding yourself to the CONTRIBUTORS file +alongside this one, and is a requirement for contributing to this project. + +### Code of Conduct + +Please be aware of and follow the +[Momentum Code of Coduct](https://github.com/MetOffice/Momentum/blob/main/docs/CODE_OF_CONDUCT.md) + +### Working Practices + +This project is managed as part of the Simulation Systems group of repositories. + +Please follow the Simulation Systems +[Working Practices.](https://metoffice.github.io/simulation-systems/index.html) + +Questions are encouraged in the Simulation Systems +[Discussions.](https://github.com/MetOffice/simulation-systems/discussions) + +Please be aware of and follow the Simulation Systems +[AI Policy.](https://metoffice.github.io/simulation-systems/FurtherDetails/ai.html) From 22c77c68278866b68a9f68295af4d75ba4cbc2db Mon Sep 17 00:00:00 2001 From: James Bruten <109733895+james-bruten-mo@users.noreply.github.com> Date: Tue, 9 Dec 2025 13:54:41 +0000 Subject: [PATCH 7/9] update action name (#9) --- .github/workflows/cla-check.yaml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cla-check.yaml b/.github/workflows/cla-check.yaml index cc76b84..b5d04c1 100644 --- a/.github/workflows/cla-check.yaml +++ b/.github/workflows/cla-check.yaml @@ -1,12 +1,10 @@ -name: CLA Check +name: Legal on: - pull_request_target: + pull_request_target: jobs: - cla_check: - uses: MetOffice/growss/.github/workflows/cla-check.yaml@main - - # Optional - with: - runner: 'ubuntu-24.04' + cla: + uses: MetOffice/growss/.github/workflows/cla-check.yaml@develop + with: + runner: 'ubuntu-24.04' From 92c4dde91403bfcb402a93eaf27a9e593c0b141e Mon Sep 17 00:00:00 2001 From: James Bruten <109733895+james-bruten-mo@users.noreply.github.com> Date: Tue, 9 Dec 2025 15:03:53 +0000 Subject: [PATCH 8/9] update default group (#10) --- rose-stem/flow.cylc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rose-stem/flow.cylc b/rose-stem/flow.cylc index f968aa7..564462d 100644 --- a/rose-stem/flow.cylc +++ b/rose-stem/flow.cylc @@ -21,7 +21,7 @@ {% elif g is defined %} {% set RUN_NAMES = g %} {% else %} - {% set RUN_NAMES = ["developer"] %} + {% set RUN_NAMES = ["scripts"] %} {% endif %} {% do LOG.info("Running groups: "~RUN_NAMES|join(' ')) %} From 6293f6537fc2a4b9783e7561193f42bb43f4f08b Mon Sep 17 00:00:00 2001 From: James Bruten <109733895+james-bruten-mo@users.noreply.github.com> Date: Tue, 9 Dec 2025 15:53:20 +0000 Subject: [PATCH 9/9] Change CLA checker to main (#11) --- .github/workflows/cla-check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cla-check.yaml b/.github/workflows/cla-check.yaml index b5d04c1..3d28d73 100644 --- a/.github/workflows/cla-check.yaml +++ b/.github/workflows/cla-check.yaml @@ -5,6 +5,6 @@ on: jobs: cla: - uses: MetOffice/growss/.github/workflows/cla-check.yaml@develop + uses: MetOffice/growss/.github/workflows/cla-check.yaml@main with: runner: 'ubuntu-24.04'