From 262fd6e18484c5caeecb5b7d2226eb8041ca7be5 Mon Sep 17 00:00:00 2001 From: "Sode, Adedamola (DLSLtd,RAL,LSCI)" Date: Fri, 7 Feb 2025 16:53:27 +0000 Subject: [PATCH 1/3] Seperated dataclasses in datatypes from guibuilder Rebasing phoebus class redesign with changes made to testing and dependencies. --- .pdm-python | 1 + .python-version | 1 + src/phoebus_guibuilder/__init__.py | 5 +- src/phoebus_guibuilder/__main__.py | 7 +- src/phoebus_guibuilder/datatypes.py | 67 +++++++ src/phoebus_guibuilder/guibuilder.py | 288 ++++++++++++--------------- tests/test_objects.py | 9 +- 7 files changed, 204 insertions(+), 174 deletions(-) create mode 100644 .pdm-python create mode 100644 .python-version create mode 100644 src/phoebus_guibuilder/datatypes.py diff --git a/.pdm-python b/.pdm-python new file mode 100644 index 00000000..e3ba2ef2 --- /dev/null +++ b/.pdm-python @@ -0,0 +1 @@ +/workspaces/guibuilder/.venv/bin/python \ No newline at end of file diff --git a/.python-version b/.python-version new file mode 100644 index 00000000..e4fba218 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/src/phoebus_guibuilder/__init__.py b/src/phoebus_guibuilder/__init__.py index 8df6208a..a80af651 100644 --- a/src/phoebus_guibuilder/__init__.py +++ b/src/phoebus_guibuilder/__init__.py @@ -8,4 +8,7 @@ from ._version import __version__ -__all__ = ["__version__"] +from phoebus_guibuilder.datatypes import Beamline, Component, Entry +from phoebus_guibuilder.guibuilder import Guibuilder + +__all__ = ["__version__", "Beamline", "Component", "Entry", "Guibuilder"] diff --git a/src/phoebus_guibuilder/__main__.py b/src/phoebus_guibuilder/__main__.py index c73b9103..14d8b93d 100644 --- a/src/phoebus_guibuilder/__main__.py +++ b/src/phoebus_guibuilder/__main__.py @@ -2,8 +2,10 @@ from argparse import ArgumentParser from collections.abc import Sequence +from fileinput import filename -from . import __version__, guibuilder +from . import __version__ +from .guibuilder import Guibuilder __all__ = ["main"] @@ -20,7 +22,8 @@ def main(args: Sequence[str] | None = None) -> None: ) _args = parser.parse_args(args) - guibuilder.main(_args.filename) + gb = Guibuilder(_args.filename) + gb.extract_from_create_gui() if __name__ == "__main__": diff --git a/src/phoebus_guibuilder/datatypes.py b/src/phoebus_guibuilder/datatypes.py new file mode 100644 index 00000000..0055eda3 --- /dev/null +++ b/src/phoebus_guibuilder/datatypes.py @@ -0,0 +1,67 @@ +import re +from dataclasses import dataclass + + +@dataclass +class Beamline: + dom: str + desc: str + + +@dataclass +class Entry: + type: str + DESC: str | None + P: str + M: str | None + R: str | None + + +@dataclass +class Component: + name: str + desc: str + prefix: str + filename: str | None = None + + def __post_init__(self): + self._extract_p_and_r() + + def __repr__(self) -> str: + return f"Component(name={self.name}, desc={self.desc}, \ + prefix={self.P}, suffix={self.R}, filename={self.filename})" + + def _extract_p_and_r(self): + pattern = re.compile( + r""" + ^ # start of string + (?= # lookahead to ensure the following pattern matches + [A-Za-z0-9-]{14,16} # match 14 to 16 alphanumeric characters or hyphens + [:A-Za-z0-9]* # match zero or more colons or alphanumeric characters + [.A-Za-z0-9] # match a dot or alphanumeric character + ) + (?!.*--) # negative lookahead to ensure no double hyphens + (?!.*:\..) # negative lookahead to ensure no colon followed by a dot + ( # start of capture group 1 + (?:[A-Za-z0-9]{2,5}-){3} # match 2 to 5 alphanumeric characters followed + # by a hyphen, repeated 3 times + [\d]* # match zero or more digits + [^:]? # match zero or one non-colon character + ) + (?::([a-zA-Z0-9:]*))? # match zero or one colon followed by zero or more + # alphanumeric characters or colons (capture group 2) + (?:\.([a-zA-Z0-9]+))? # match zero or one dot followed by one or more + # alphanumeric characters (capture group 3) + $ # end of string + """, + re.VERBOSE, + ) + + match = re.match(pattern, self.prefix) + if match: + self.P: str = match.group(1) + self.R: str = match.group(2) + # TODO: Is this needed? + self.attribute: str | None = match.group(3) + else: + raise AttributeError(f"No valid PV prefix found for {self.name}.") diff --git a/src/phoebus_guibuilder/guibuilder.py b/src/phoebus_guibuilder/guibuilder.py index c3be9e46..5e6ecb5a 100644 --- a/src/phoebus_guibuilder/guibuilder.py +++ b/src/phoebus_guibuilder/guibuilder.py @@ -1,171 +1,129 @@ import os -import pprint import re -from dataclasses import dataclass +import sys import yaml -pp = pprint.PrettyPrinter() - - -@dataclass -class Beamline: - dom: str - desc: str - - -@dataclass -class Entry: - type: str - DESC: str | None - P: str - M: str | None - R: str | None - - -@dataclass -class Component: - name: str - desc: str - prefix: str - filename: str | None = None - - def __post_init__(self): - self._extract_p_and_r() - - def __repr__(self) -> str: - return f"Component(name={self.name}, desc={self.desc}, prefix={self.P}, \ -suffix={self.R}, filename={self.filename})" - - def _extract_p_and_r(self): - pattern = re.compile( - r""" - ^ # start of string - (?= # lookahead to ensure the following pattern matches - [A-Za-z0-9-]{14,16} # match 14 to 16 alphanumeric characters or hyphens - [:A-Za-z0-9]* # match zero or more colons or alphanumeric characters - [.A-Za-z0-9] # match a dot or alphanumeric character - ) - (?!.*--) # negative lookahead to ensure no double hyphens - (?!.*:\..) # negative lookahead to ensure no colon followed by a dot - ( # start of capture group 1 - (?:[A-Za-z0-9]{2,5}-){3} # match 2 to 5 alphanumeric characters followed - # by a hyphen, repeated 3 times - [\d]* # match zero or more digits - [^:]? # match zero or one non-colon character - ) - (?::([a-zA-Z0-9:]*))? # match zero or one colon followed by zero or more - # alphanumeric characters or colons (capture group 2) - (?:\.([a-zA-Z0-9]+))? # match zero or one dot followed by one or more - # alphanumeric characters (capture group 3) - $ # end of string - """, - re.VERBOSE, - ) - - match = re.match(pattern, self.prefix) - if match: - self.P: str = match.group(1) - self.R: str = match.group(2) - # TODO: Is this needed? - self.attribute: str | None = match.group(3) - else: - raise AttributeError(f"No valid PV prefix found for {self.name}.") - - -def main(filename: str): - components: list[Component] = [] # TODO Manage global lists better - - with open(filename) as f: - conf = yaml.safe_load(f) - - bl: dict[str, str] = conf["beamline"] - comps: dict[str, dict[str, str]] = conf["components"] - - beamline = Beamline(**bl) - - for key, comp in comps.items(): - components.append(Component(key, **comp)) - - print("BEAMLINE:") - pp.pprint(beamline) - - print("") - print("COMPONENTS") - pp.pprint(components) - - -##################################################### -# TODO Functionality should be in phoebusguibuilder class -# class Phoebusguibuilder(beamline: Beamline, components: list[Component]): - - -def find_services_folders(beamline: Beamline, components: list[Component]): - services_directory = ( - beamline.dom + "-services/services" - ) # TODO: rm hardcoding, map to services. - path = f"/dls/science/users/uns32131/{services_directory}" - files = os.listdir(path) - - # Attempting to match the prefix to the files in the services directory - pattern = "^(.*)-(.*)-(.*)" - - for component in components: - domain: re.Match[str] | None = re.match(pattern, component.P) - assert domain is not None, "Empty Prefix Field" - - for file in files: - match = re.match(pattern, file) - if match: - if match.group(1) == domain.group(1).lower(): - if os.path.exists(f"{path}/{file}/config/ioc.yaml"): - valid_entities = extract_valid_entities( - ioc_yaml=f"{path}/{file}/config/ioc.yaml", - component=component, +from phoebus_guibuilder.datatypes import Beamline, Component, Entry + + +class Guibuilder: + """ + This class provides the functionality to process the required + create_gui.yaml file into screens mapped from ioc.yaml and + gui_map.yaml files. + + """ + + def __init__(self, create_gui: str): + self.components: list[Component] + + self.beamline: Beamline + + self.valid_entities: list[Entry] = [] + + self.create_gui: str = create_gui + + def extract_from_create_gui( + self, + ): + """ + Extracts from the create_gui.yaml file to generate + the required Beamline and components structures. + """ + + with open(self.create_gui) as f: + conf = yaml.safe_load(f) + + bl: dict[str, str] = conf["beamline"] + comps: dict[str, dict[str, str]] = conf["components"] + + self.beamline = Beamline(**bl) + + for key, comp in comps.items(): + self.components.append(Component(key, **comp)) + + def find_services_folders( + self, + ): + """ + Finds the related folders in the services directory + and extracts the related entites with the matching prefixes + """ + + services_directory = ( + self.beamline.dom + "-services/services" + ) # TODO: rm hardcoding, map to services. + path = f"{services_directory}" + files = os.listdir(path) + + # Attempting to match the prefix to the files in the services directory + pattern = "^(.*)-(.*)-(.*)" + + for component in self.components: + domain: re.Match[str] | None = re.match(pattern, component.P) + assert domain is not None, "Empty Prefix Field" + + for file in files: + match = re.match(pattern, file) + if match: + if match.group(1) == domain.group(1).lower(): + if os.path.exists(f"{path}/{file}/config/ioc.yaml"): + self.extract_valid_entities( + ioc_yaml=f"{path}/{file}/config/ioc.yaml", + component=component, + ) + else: + print(f"No ioc.yaml file for service: {file}") + + def extract_valid_entities(self, ioc_yaml: str, component: Component): + """ + Extracts the entities in ioc.yaml matching the defined prefix + """ + + entities: list[dict[str, str]] = [] + component_match = f"{component.P}:{component.R}" + + with open(ioc_yaml) as ioc: + conf = yaml.safe_load(ioc) + entities = conf["entities"] + for entity in entities: + if ( + "P" in entity.keys() and entity["P"] == component_match + ): # the suffix could be M, could be R + self.valid_entities.append( + Entry( + type=entity["type"], + DESC=None, + P=entity["P"], + M=None, + R=None, ) - return valid_entities - else: - print(f"No ioc.yaml file for service: {file}") - - -def extract_valid_entities(ioc_yaml: str, component: Component) -> list[Entry]: - print(type(ioc_yaml)) - entities: list[dict[str, str]] = [] - valid_entities: list[Entry] = [] - component_match = f"{component.P}:{component.R}" - with open(ioc_yaml) as ioc: - conf = yaml.safe_load(ioc) - entities = conf["entities"] - for entity in entities: - if ( - "P" in entity.keys() and entity["P"] == component_match - ): # the suffix could be M, could be R - valid_entities.append( - Entry(type=entity["type"], DESC=None, P=entity["P"], M=None, R=None) - ) - - return valid_entities - - -def gui_map(entrys: list[Entry]): - gui_map = "/dls/science/users/uns32131/BLGui/BLGuiApp/opi/bob/gui_map.yaml" - - with open(gui_map) as map: - conf = yaml.safe_load(map) - - for entry in entrys: - print(entry.type) - if conf[entry.type]: - print( - conf[entry.type]["file"] - ) # Find correct .bob file, and injet macros - # TODO: create a copy of the file, and replace the required macros - # TODO: return the file to guibuilder - - else: - print("No BOB available") - - -# find_services_folders() -# print(valid_entities) -# gui_map(valid_entities) + ) + + def gui_map(self, entrys: list[Entry]): + """ + Maps the valid entities from the ioc.yaml file + to the required screen in gui_map.yaml + """ + + gui_map = "/BLGui/BLGuiApp/opi/bob/gui_map.yaml" + + with open(gui_map) as map: + conf = yaml.safe_load(map) + + for entry in entrys: + print(entry.type) + if conf[entry.type]: + print( + conf[entry.type]["file"] + ) # Find correct .bob file, and injet macros + # TODO: create a copy of the file, and replace the required macros + # TODO: return the file to guibuilder + + else: + print("No BOB available") + + +if __name__ == "__main__": + Guibuilder(sys.argv[1]) diff --git a/tests/test_objects.py b/tests/test_objects.py index acb0357a..e2ee0f80 100644 --- a/tests/test_objects.py +++ b/tests/test_objects.py @@ -1,6 +1,6 @@ import pytest -from phoebus_guibuilder.guibuilder import Beamline, Component +from phoebus_guibuilder.datatypes import Beamline, Component @pytest.fixture @@ -28,11 +28,8 @@ def test_component_object(component): def test_component_repr(component): - assert ( - str(component) - == "Component(name=TESTDEV, desc=Test Device, \ -prefix=BL01T-EA-TEST-02, suffix=None, filename=None)" - ) + assert (str(component)== "Component(name=TESTDEV, desc=Test Device, \ + prefix=BL01T-EA-TEST-02, suffix=None, filename=None)") def test_component_bad_prefix(): From 957a72ac3cab3b2ce1384b6e5d28cedf4f92753a Mon Sep 17 00:00:00 2001 From: "Sode, Adedamola (DLSLtd,RAL,LSCI)" Date: Thu, 20 Feb 2025 12:12:55 +0000 Subject: [PATCH 2/3] guibuilder.py: create_gui to create_gui_yaml, removed __main__, linting with ruff --- src/phoebus_guibuilder/__init__.py | 4 ++-- src/phoebus_guibuilder/__main__.py | 1 - src/phoebus_guibuilder/guibuilder.py | 9 ++------- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/phoebus_guibuilder/__init__.py b/src/phoebus_guibuilder/__init__.py index a80af651..e2e97e62 100644 --- a/src/phoebus_guibuilder/__init__.py +++ b/src/phoebus_guibuilder/__init__.py @@ -6,9 +6,9 @@ Version number as calculated by poetry-dynamic-versioning """ -from ._version import __version__ - from phoebus_guibuilder.datatypes import Beamline, Component, Entry from phoebus_guibuilder.guibuilder import Guibuilder +from ._version import __version__ + __all__ = ["__version__", "Beamline", "Component", "Entry", "Guibuilder"] diff --git a/src/phoebus_guibuilder/__main__.py b/src/phoebus_guibuilder/__main__.py index 14d8b93d..a62c8db1 100644 --- a/src/phoebus_guibuilder/__main__.py +++ b/src/phoebus_guibuilder/__main__.py @@ -2,7 +2,6 @@ from argparse import ArgumentParser from collections.abc import Sequence -from fileinput import filename from . import __version__ from .guibuilder import Guibuilder diff --git a/src/phoebus_guibuilder/guibuilder.py b/src/phoebus_guibuilder/guibuilder.py index 5e6ecb5a..fb65183b 100644 --- a/src/phoebus_guibuilder/guibuilder.py +++ b/src/phoebus_guibuilder/guibuilder.py @@ -1,6 +1,5 @@ import os import re -import sys import yaml @@ -15,14 +14,14 @@ class Guibuilder: """ - def __init__(self, create_gui: str): + def __init__(self, create_gui_yaml: str): self.components: list[Component] self.beamline: Beamline self.valid_entities: list[Entry] = [] - self.create_gui: str = create_gui + self.create_gui: str = create_gui_yaml def extract_from_create_gui( self, @@ -123,7 +122,3 @@ def gui_map(self, entrys: list[Entry]): else: print("No BOB available") - - -if __name__ == "__main__": - Guibuilder(sys.argv[1]) From 09e02e10a40ecccf3919255c963443a8db628072 Mon Sep 17 00:00:00 2001 From: "Sode, Adedamola (DLSLtd,RAL,LSCI)" Date: Fri, 21 Feb 2025 11:26:08 +0000 Subject: [PATCH 3/3] Adapted guibuilder ioc_yaml for github-api, added tests --- src/phoebus_guibuilder/git_utilities.py | 119 ++++++++++++++++++++++++ src/phoebus_guibuilder/guibuilder.py | 44 +++------ tests/test_guibuilder.py | 19 ++++ 3 files changed, 153 insertions(+), 29 deletions(-) create mode 100644 src/phoebus_guibuilder/git_utilities.py create mode 100644 tests/test_guibuilder.py diff --git a/src/phoebus_guibuilder/git_utilities.py b/src/phoebus_guibuilder/git_utilities.py new file mode 100644 index 00000000..a9e44084 --- /dev/null +++ b/src/phoebus_guibuilder/git_utilities.py @@ -0,0 +1,119 @@ +import base64 +import re + +import requests + + +class GitYaml: + def __init__(self, dom: str): + self.pattern: str = "^(.*)-(.*)-(.*)" + self.dom: str = dom + self.git_ref: str = ( + f"https://api.github.com/repos/epics-containers/{dom}-services/git/refs" + ) + + def re_group(self, component: str) -> str | None: + match: re.Match[str] | None = re.match(self.pattern, component) + + if match: + return match.group(1) + + def get_json_from_url(self, url: str) -> dict | None: + try: + response = requests.get(url) + print(response) + response.raise_for_status() + if response: + return response.json() + except requests.exceptions.HTTPError as err: + print(err.response.text) + return err.response.json() + + def get_yaml(self, url) -> str | None: + data: dict | None = self.get_json_from_url(url) + if data is not None: + base64_content = data["content"] + decoded_content = base64.b64decode(base64_content).decode("utf-8") + + return decoded_content + + def fetch_matches( + self, + tree: dict, + task: str, + prefix: str = "", + option: str = "", + ) -> str | None: + if task == "ref": + matching_refs = [ + item + for item in tree + if isinstance(item, dict) and item.get("ref") == "refs/heads/main" + ] + return matching_refs[0]["object"]["sha"] + elif task == "services": + tree = tree["tree"] + matching_url = [ + item + for item in tree + if isinstance(item, dict) and item.get("path") == "services" + ] + print(f"DEBUG1:{matching_url}") + return matching_url[0]["url"] + + elif task == "subfolder": + tree = tree["tree"] + matching_folders = [ + item + for item in tree + if isinstance(item, dict) + and self.re_group(str(item.get("path"))) == prefix.lower() + ] + if matching_folders: + return matching_folders[0]["url"] + + elif task == "config": + tree = tree["tree"] + config = [ + item + for item in tree + if isinstance(item, dict) and item.get("path") == "config" + ] + return config[0]["url"] + + else: + answer = [ + item + for item in tree + if isinstance(item, dict) and item.get(option) == task + ] + return answer[0]["url"] + + def fetch_ioc_yaml( + self, + ): + commit_hashes: dict | None = self.get_json_from_url(self.git_ref) + + assert commit_hashes is not None, "Could not pull commit hashes" + if commit_hashes["message"]: + return + + main_hash = self.fetch_matches(commit_hashes, "ref") + main_url = f"https://api.github.com/repos/epics-containers/{self.dom}-services/git/trees/{main_hash}" + main_tree: dict | None = self.get_json_from_url(main_url) + assert main_tree is not None, "Could not obtain main json tree" + + services_url: str | None = self.fetch_matches(main_tree, "services") + assert services_url is not None, "Could not fetch services url" + + services_json: dict | None = self.get_json_from_url(services_url) + assert services_json is not None, "Could not fetch services json tree" + config_url = self.fetch_matches(services_json, "subfolder") + + if config_url is not None: + config_json: dict | None = self.get_json_from_url(config_url) + assert config_json is not None, "Could not fetch config json tree" + + ioc_yaml_url = self.fetch_matches(config_json, "config") + + return self.get_yaml(ioc_yaml_url) diff --git a/src/phoebus_guibuilder/guibuilder.py b/src/phoebus_guibuilder/guibuilder.py index fb65183b..56eed083 100644 --- a/src/phoebus_guibuilder/guibuilder.py +++ b/src/phoebus_guibuilder/guibuilder.py @@ -1,9 +1,7 @@ -import os -import re - import yaml from phoebus_guibuilder.datatypes import Beamline, Component, Entry +from phoebus_guibuilder.git_utilities import GitYaml class Guibuilder: @@ -15,7 +13,7 @@ class Guibuilder: """ def __init__(self, create_gui_yaml: str): - self.components: list[Component] + self.components: list[Component] = [] self.beamline: Beamline @@ -42,38 +40,24 @@ def extract_from_create_gui( for key, comp in comps.items(): self.components.append(Component(key, **comp)) - def find_services_folders( + def find_services_extract_ioc( self, ): """ - Finds the related folders in the services directory + Finds the related folders in the services repo and extracts the related entites with the matching prefixes """ - services_directory = ( - self.beamline.dom + "-services/services" - ) # TODO: rm hardcoding, map to services. - path = f"{services_directory}" - files = os.listdir(path) - - # Attempting to match the prefix to the files in the services directory - pattern = "^(.*)-(.*)-(.*)" - for component in self.components: - domain: re.Match[str] | None = re.match(pattern, component.P) - assert domain is not None, "Empty Prefix Field" - - for file in files: - match = re.match(pattern, file) - if match: - if match.group(1) == domain.group(1).lower(): - if os.path.exists(f"{path}/{file}/config/ioc.yaml"): - self.extract_valid_entities( - ioc_yaml=f"{path}/{file}/config/ioc.yaml", - component=component, - ) - else: - print(f"No ioc.yaml file for service: {file}") + print(component.P) + ioc_yaml = GitYaml(self.beamline.dom).fetch_ioc_yaml() + if ioc_yaml is not None: + self.extract_valid_entities( + ioc_yaml, + component=component, + ) + else: + print("Cannot find the yaml file, check the repo") def extract_valid_entities(self, ioc_yaml: str, component: Component): """ @@ -86,6 +70,7 @@ def extract_valid_entities(self, ioc_yaml: str, component: Component): with open(ioc_yaml) as ioc: conf = yaml.safe_load(ioc) entities = conf["entities"] + print(entities) for entity in entities: if ( "P" in entity.keys() and entity["P"] == component_match @@ -99,6 +84,7 @@ def extract_valid_entities(self, ioc_yaml: str, component: Component): R=None, ) ) + print(self.valid_entities) def gui_map(self, entrys: list[Entry]): """ diff --git a/tests/test_guibuilder.py b/tests/test_guibuilder.py new file mode 100644 index 00000000..e9fdfe5c --- /dev/null +++ b/tests/test_guibuilder.py @@ -0,0 +1,19 @@ +from phoebus_guibuilder.guibuilder import Guibuilder + + +def test_guibuilder(): + gb = Guibuilder("./example/create_gui.yaml") + gb.extract_from_create_gui() + print(gb.create_gui) + print("\n") + print(gb.components) + print("\n") + print(gb.beamline) + print("\n") + + +def test_gb_extract_services(): + gb = Guibuilder("./example/create_gui.yaml") + gb.extract_from_create_gui() + gb.find_services_extract_ioc() + print(gb.valid_entities)