From d1457eff0ae68189d40b3f39fc2fabd2caf50949 Mon Sep 17 00:00:00 2001 From: Rafael Vescovi Date: Wed, 4 Jun 2025 16:01:52 -0500 Subject: [PATCH 1/2] init based on bcda template --- .gitignore | 1 + pyproject.toml | 5 +- .../demo_instrument/configs/tiled_config.yml | 15 ++ src/apsbits/demo_instrument/startup.py | 2 + src/apsbits/tiled/__init__.py | 1 + src/apsbits/tiled/custom.py | 65 +++++++ src/apsbits/tiled/discover_more_catalogs.py | 64 +++++++ src/apsbits/tiled/ignore_data.py | 21 +++ src/apsbits/tiled/image_data.py | 173 ++++++++++++++++++ src/apsbits/tiled/spec_data.py | 161 ++++++++++++++++ src/apsbits/tiled/synApps_mda.py | 133 ++++++++++++++ 11 files changed, 639 insertions(+), 2 deletions(-) create mode 100644 src/apsbits/demo_instrument/configs/tiled_config.yml create mode 100644 src/apsbits/tiled/__init__.py create mode 100644 src/apsbits/tiled/custom.py create mode 100644 src/apsbits/tiled/discover_more_catalogs.py create mode 100644 src/apsbits/tiled/ignore_data.py create mode 100644 src/apsbits/tiled/image_data.py create mode 100644 src/apsbits/tiled/spec_data.py create mode 100644 src/apsbits/tiled/synApps_mda.py diff --git a/.gitignore b/.gitignore index 7e802caf..64e778cc 100644 --- a/.gitignore +++ b/.gitignore @@ -176,3 +176,4 @@ existing_plans_and_devices.yaml # Local Run Engine metadata dictionary .re_md_dict.yml +*.db* diff --git a/pyproject.toml b/pyproject.toml index 2510a898..f963e6c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,13 +33,14 @@ classifiers = [ "Topic :: Utilities", ] dependencies = [ - "apstools >= 1.7.2", + "apstools", "bluesky-queueserver-api", "bluesky-queueserver", "bluesky-widgets", "bluesky", "caproto", - "databroker ==1.2.5", + "databroker", + "tiled[server]", "guarneri", "ipython", "jupyterlab", diff --git a/src/apsbits/demo_instrument/configs/tiled_config.yml b/src/apsbits/demo_instrument/configs/tiled_config.yml new file mode 100644 index 00000000..fec252aa --- /dev/null +++ b/src/apsbits/demo_instrument/configs/tiled_config.yml @@ -0,0 +1,15 @@ +# config.yml + +# tiled serve config --public --host 0.0.0.0 config.yml + +# For security when using tiled server to write bluesky runs, +# set the API key by setting env var +# TILED_API_KEY rather than putting it in code. + +trees: + + - path: demo_instrument + tree: databroker.mongo_normalized:Tree.from_uri + args: + # for unsecured access + uri: mongodb://localhost:27017/demo_instrument diff --git a/src/apsbits/demo_instrument/startup.py b/src/apsbits/demo_instrument/startup.py index 6efa0c31..3b68103d 100644 --- a/src/apsbits/demo_instrument/startup.py +++ b/src/apsbits/demo_instrument/startup.py @@ -109,3 +109,5 @@ # Setup baseline stream with connect=False is default # Devices with the label 'baseline' will be added to the baseline stream. setup_baseline_stream(sd, oregistry, connect=False) + +from .plans import * \ No newline at end of file diff --git a/src/apsbits/tiled/__init__.py b/src/apsbits/tiled/__init__.py new file mode 100644 index 00000000..4e9fcc7d --- /dev/null +++ b/src/apsbits/tiled/__init__.py @@ -0,0 +1 @@ +"""Common tiled utilities.""" diff --git a/src/apsbits/tiled/custom.py b/src/apsbits/tiled/custom.py new file mode 100644 index 00000000..3128f0e3 --- /dev/null +++ b/src/apsbits/tiled/custom.py @@ -0,0 +1,65 @@ +""" +Custom handling for data file types not recognized by tiled. + +https://blueskyproject.io/tiled/how-to/read-custom-formats.html +""" + +import pathlib + +import h5py +from punx.utils import isHdf5FileObject, isNeXusFile +from spec2nexus.spec import is_spec_file_with_header +from spec_data import MIMETYPE as SPEC_MIMETYPE + +FILE_OF_UNRECOGNIZED_FILE_TYPES = "/tmp/unrecognized_files.txt" +HDF5_MIMETYPE = "application/x-hdf5" + + +def isHdf5(filename): + try: + with h5py.File(filename, "r") as fp: + return isHdf5FileObject(fp) + except Exception: + pass + return False + + +def isNeXus(filename): + try: + return isNeXusFile(filename) + except Exception: + pass + return False + + +mimetype_table = { + is_spec_file_with_header: SPEC_MIMETYPE, # spec2nexus.spec.is_spec_file_with_header + isNeXus: HDF5_MIMETYPE, # punx.utils.isNeXusFile + isHdf5: HDF5_MIMETYPE, # punx.utils.isHdf5FileObject +} + + +def detect_mimetype(filename, mimetype): + filename = pathlib.Path(filename) + if "/.log" in str(filename).lower(): + mimetype = "text/plain" + elif ".log" in filename.name.lower(): + mimetype = "text/plain" + + if mimetype is None: + # When tiled has not already recognized the mimetype. + mimetype = "text/csv" # the default + for tester, mtype in mimetype_table.items(): + # iterate through our set of known types + if tester(filename): + mimetype = mtype + break + if filename.name == "README": + mimetype = "text/readme" + + if mimetype is None: + with open(FILE_OF_UNRECOGNIZED_FILE_TYPES, "a") as fp: + # TODO: What's the point of writing mimetype here? It's `None`! + fp.write(f"{mimetype} {filename}\n") + + return mimetype diff --git a/src/apsbits/tiled/discover_more_catalogs.py b/src/apsbits/tiled/discover_more_catalogs.py new file mode 100644 index 00000000..168c193d --- /dev/null +++ b/src/apsbits/tiled/discover_more_catalogs.py @@ -0,0 +1,64 @@ +""" +Identify new MongoDB catalogs for the tiled server. + +This code prints additional lines that could be added +to the tiled `config.yml` file. The lines describe catalogs +with known intake descriptions that are not already configured +in the `config.yml` file. +""" + +import pathlib + +import yaml + + +def tiled_test(): + from tiled.client import from_uri + + client = from_uri("http://otz:8000", "dask") + cat = client["6idd"] + print(f"{cat=}") + + +def read_intake_yaml(file) -> dict: + with open(file) as f: + db_cfg = yaml.load(f, Loader=yaml.Loader) + return db_cfg["sources"] + + +def main(): + home = pathlib.Path.home() + # print(f"{home=}") + databroker_configs = home / ".local" / "share" / "intake" + # print(f"exists:{databroker_configs.exists()} {databroker_configs}") + + master = { + k: v + for intake_yml in databroker_configs.iterdir() + if intake_yml.is_file() and intake_yml.suffix == ".yml" + for k, v in read_intake_yaml(intake_yml).items() + } + + local_config = pathlib.Path(__file__).parent / "config.yml" + # print(f"exists:{local_config.exists()} {local_config}") + with open(local_config) as f: + config = yaml.load(f, Loader=yaml.Loader) + trees = {tree["path"]: tree for tree in config.get("trees", {})} + + new_entries = [] + for k, source in master.items(): + if k not in trees: + if source.get("driver") == "bluesky-mongo-normalized-catalog": + uri = source.get("args", {}).get("metadatastore_db") + if uri is not None: + entry = dict( + path=k, + tree="databroker.mongo_normalized:Tree.from_uri", + args=dict(uri=uri), + ) + new_entries.append(entry) + print(yaml.dump(new_entries, indent=4)) + + +if __name__ == "__main__": + main() diff --git a/src/apsbits/tiled/ignore_data.py b/src/apsbits/tiled/ignore_data.py new file mode 100644 index 00000000..ff607b18 --- /dev/null +++ b/src/apsbits/tiled/ignore_data.py @@ -0,0 +1,21 @@ +import numpy +from tiled.adapters.array import ArrayAdapter +from tiled.adapters.mapping import MapAdapter +from tiled.structures.core import Spec as TiledSpec + +IGNORE_SPECIFICATION = TiledSpec("ignore", version="1.0") + + +def read_ignore(filename, **kwargs): + arrays = dict( + ignore=ArrayAdapter.from_array( + numpy.array([0, 0]), + metadata=dict(ignore="placeholder, ignore"), + specs=[IGNORE_SPECIFICATION], + ) + ) + return MapAdapter( + arrays, + metadata=dict(filename=str(filename), purpose="ignore this file's contents"), + specs=[IGNORE_SPECIFICATION], + ) diff --git a/src/apsbits/tiled/image_data.py b/src/apsbits/tiled/image_data.py new file mode 100644 index 00000000..b7734e01 --- /dev/null +++ b/src/apsbits/tiled/image_data.py @@ -0,0 +1,173 @@ +""" +Read a variety of image file formats as input for tiled. +""" + +import os +import pathlib + +import numpy +import yaml +from PIL import Image +from PIL.TiffImagePlugin import IFDRational +from tiled.adapters.array import ArrayAdapter +from tiled.adapters.mapping import MapAdapter +from tiled.structures.core import Spec as TiledSpec + +from ignore_data import IGNORE_SPECIFICATION + +ROOT = pathlib.Path(__file__).parent + +EXTENSIONS = [] +# https://mimetype.io/all-types#image +MIMETYPES = """ + image/bmp + image/gif + image/jpeg + image/png + image/tiff + image/vnd.microsoft.icon + image/webp +""".split() +# TODO: image/avif not handled by PIL +# TODO: image/svg+xml not handled by PIL + +EMPTY_ARRAY = numpy.array([0, 0]) +IMAGE_FILE_SPECIFICATION = TiledSpec("image_file", version="1.0") + + +def interpret_IFDRational(data): + if not isinstance(data, IFDRational): + raise TypeError(f"{data} is not of type {IFDRational.__class__}") + attrs = "numerator denominator imag".split() + md = {k: getattr(data, k) for k in attrs} + md["real"] = float(data.numerator) / data.denominator + return md + + +def interpret_exif(image): + from PIL.ExifTags import TAGS + + exif = image.getexif() + md = {} + for tag_id in exif: + # get the tag name, instead of human unreadable tag id + tag = TAGS.get(tag_id, tag_id) + data = exif.get(tag_id) + # decode bytes + if isinstance(data, bytes): + data = data.decode() + if isinstance(data, IFDRational): + data = interpret_IFDRational(data) + md[tag] = data + return md + + +def image_metadata(image): + attrs = """ + bits + filename + format + format_description + is_animated + layer + layers + mode + n_frames + size + text + """.split() + md = {k: getattr(image, k) for k in attrs if hasattr(image, k)} + + if len(image.info) > 0: + md["info"] = {} + md["info"].update(image.info) + info = md.get("info") + if info is not None: + for k in "exif icc_profile xmp".split(): + if k in info: + info.pop(k) + for k in "dpi resolution".split(): + # fmt: off + if k in info: + items = [] + for data in info[k]: + if isinstance(data, IFDRational): + v = interpret_IFDRational(data) + else: + v = data + items.append(v) + info[k] = tuple(items) + # fmt: on + value = info.get("version") + if isinstance(value, bytes): + info["version"] = value.decode() + value = info.get("extension") + if isinstance(value, tuple) and isinstance(value[0], bytes): + info["extension"] = (value[0].decode(), value[1]) + + # print(yaml.dump(md)) + + exif = interpret_exif(image) + if len(exif) > 0: + md["exif"] = exif + + md["extrema"] = image.getextrema() + + return md + + +def read_image(filename, **kwargs): + fn = pathlib.Path(filename).name + try: + if not os.path.isfile(filename): + raise TypeError(f"'{filename}' is not a file.") + image = Image.open(filename) + md = image_metadata(image) + + # # special cases + # if image.format == "AVIF": + # pass + + im = image.getdata() + pixels = list(im) # 1-D array of int or tuple + shape = list(reversed(im.size)) + if im.bands > 1: + shape.append(im.bands) + pixels = numpy.array(pixels).reshape(shape) + if len(shape) > 2: + pixels = numpy.moveaxis(pixels, -1, 0) # put the colors first + return ArrayAdapter.from_array( + pixels, metadata=md, specs=[IMAGE_FILE_SPECIFICATION] + ) + + except Exception as exc: + arrays = dict( + ignore=ArrayAdapter.from_array( + numpy.array([0, 0]), + metadata=dict(ignore="placeholder, ignore"), + specs=[IGNORE_SPECIFICATION], + ) + ) + return MapAdapter( + arrays, + metadata=dict( + filename=str(filename), + exception=exc, + purpose="some problem reading this file as an image", + specs=[IGNORE_SPECIFICATION], + ), + ) + + +def main(): + testdir = ROOT / "data" / "usaxs" / "2021" + for filepath in testdir.iterdir(): + read_image(filepath) + + testdir = ROOT / "data" / "images" + for filepath in testdir.iterdir(): + read_image(filepath) + + +if __name__ == "__main__": + main() diff --git a/src/apsbits/tiled/spec_data.py b/src/apsbits/tiled/spec_data.py new file mode 100644 index 00000000..d309f698 --- /dev/null +++ b/src/apsbits/tiled/spec_data.py @@ -0,0 +1,161 @@ +"""Read the SPEC data file format.""" + +import datetime +import os + +import numpy +from spec2nexus import spec +from tiled.adapters.array import ArrayAdapter +from tiled.adapters.mapping import MapAdapter +from tiled.structures.core import Spec as TiledSpec + +EXTENSIONS = [] # no uniform standard exists, many common patterns +MIMETYPE = "text/x-spec_data" +SPEC_FILE_SPECIFICATION = TiledSpec("SPEC_file", version="1.0") +SPEC_SCAN_SPECIFICATION = TiledSpec("SPEC_scan", version="1.0") + + +def has(parent, attr): + """Cautious alternative to hasattr().""" + if attr in dir(parent): + obj = getattr(parent, attr) + return obj is not None + return False + + + +def read_diffractometer_metadata(diffractometer): + # special cases + simple_attrs = """ + UB + geometry_name + geometry_name_full + mode + sector + variant + wavelength + """.split() + # fmt: off + md = { + k: getattr(diffractometer, k) + for k in simple_attrs + if has(diffractometer, k) + } + + if has(diffractometer, "reflections"): + md["reflections"] = { + f"R{r}": refl._asdict() + for r, refl in enumerate(diffractometer.reflections) + } + if has(diffractometer, "geometry_parameters"): + md["geometry_parameters"] = { + k: dict( + key=v.key, + description=v.description, + value=v.value, + ) + for k, v in diffractometer.geometry_parameters.items() + } + if has(diffractometer, "lattice"): + md["lattice"] = diffractometer.lattice._asdict() + # fmt: on + return md + + +def read_spec_scan(scan): + try: + arrays = { + k: ArrayAdapter.from_array(numpy.array(v)) + # TODO: xref name as metadata? + for k, v in scan.data.items() + } + # fmt: off + attrs = """ + G L M S + column_first column_last + date epoch metadata positioner + scanCmd scanNum time_name + """.split() + md = { + k: getattr(scan, k) + for k in attrs + if has(scan, k) + } + if has(scan, "diffractometer"): + md.update(read_diffractometer_metadata(scan.diffractometer)) + # fmt: on + except ValueError as exc: + arrays = {} + md = dict(ValueError=exc, disposition="skipping") + return MapAdapter(arrays, metadata=md, specs=[SPEC_SCAN_SPECIFICATION]) + + +def read_spec_data(filename, **kwargs): + # kwargs has metadata known to the tiled database + if not spec.is_spec_file_with_header(filename): + raise spec.NotASpecDataFile(str(filename)) + sdf = spec.SpecDataFile(str(filename)) + md = dict( + fileName=str(sdf.fileName), + specFile=str(sdf.specFile), + ) + # header metadata (sdf.headers is a list) + if has(sdf, "headers") and len(sdf.headers) > 0: + md["headers"] = {} + for h, header in enumerate(sdf.headers, start=1): + h_md = md["headers"][f"H{h}"] = {} + for key in "date epoch counter_xref positioner_xref".split(): + if has(header, key): + h_md[key] = getattr(header, key) + if has(header, "file"): + h_md["file"] = str(header.file) + if has(header, "epoch"): + h_md["iso8601"] = f"{datetime.datetime.fromtimestamp(header.epoch)}" + if has(header, "comments") and len(header.comments) > 0: + h_md["comments"] = { + f"C{c}": comment + for c, comment in enumerate(header.comments, start=1) + } + + # fmt: off + return MapAdapter( + { + f"S{scan_number}": read_spec_scan(scan) + for scan_number, scan in sdf.scans.items() + }, + metadata=md, + specs=[SPEC_FILE_SPECIFICATION] + ) + # fmt: on + + +def developer(): + import pathlib + + # spec2nexus_data_path = ( + # pathlib.Path().home() + # / "Documents" + # / "projects" + # / "prjemian" + # / "spec2nexus" + # / "src" + # / "spec2nexus" + # / "data" + # ) + test_data_path = pathlib.Path(__file__).parent / "data" + # sixc_data_path = test_data_path / "diffractometer" / "sixc" + usaxs_data_path = test_data_path / "usaxs" / "2019" + path = usaxs_data_path + for filename in sorted(path.iterdir()): + print(f"{filename.name=}") + if not os.path.isfile(filename): + continue + try: + structure = read_spec_data(filename) + print(f"{structure}") + except spec.NotASpecDataFile: + pass + + +if __name__ == "__main__": + developer() diff --git a/src/apsbits/tiled/synApps_mda.py b/src/apsbits/tiled/synApps_mda.py new file mode 100644 index 00000000..7970e95b --- /dev/null +++ b/src/apsbits/tiled/synApps_mda.py @@ -0,0 +1,133 @@ +"""Read the synApps MDA file format.""" + +# FIXME: TypeError: read_mda() got an unexpected keyword argument 'specs' +# when browsing a MDA file + +import mda +from tiled.adapters.array import ArrayAdapter +from tiled.adapters.mapping import MapAdapter +from tiled.structures.core import Spec as TiledSpec + +EXTENSIONS = [".mda"] +MIMETYPE = "application/x-mda" +MDA_FILE_SPECIFICATION = TiledSpec("MDA_file", version="1.0") +MDA_SCAN_SPECIFICATION = TiledSpec("MDA_scan", version="1.0") + + +def as_str(v): + if isinstance(v, bytes): + return v.decode() + return v + + +def read_mda_header(mda_obj): + h_obj = mda_obj[0] + file_md = {key: h_obj[key] for key in h_obj["ourKeys"] if key != "ourKeys"} + file_md["PVs"] = { + as_str(key): dict( + desc=as_str(values[0]), + unit=as_str(values[1]), + value=values[2], + EPICS_type=mda.EPICS_types_dict.get(values[3], f"unknown #{values[3]}"), + count=values[4], + ) + for key, values in h_obj.items() + if key not in h_obj["ourKeys"] + } + if "version" in file_md: + # fix the truncation error of 1.299999... + file_md["version"] = round(file_md["version"], 2) + if len(mda_obj) != file_md["rank"] + 1: + raise ValueError(f"rank={file_md['rank']} but {len(mda_obj)=}") + + return file_md + + +def read_mda_scan_detector(detector): + md = {k: getattr(detector, k) for k in "desc fieldName number unit".split()} + md["EPICS_PV"] = as_str(detector.name) + return md["fieldName"], ArrayAdapter.from_array(detector.data, metadata=md) + + +def read_mda_scan_positioner(positioner): + md_attrs = """ + desc + fieldName + number + readback_desc + readback_name + readback_unit + step_mode + unit + """.split() + md = {k: getattr(positioner, k) for k in md_attrs} + md["readback_PV"] = md.pop("readback_name") # rename + md["EPICS_PV"] = as_str(positioner.name) + return md["fieldName"], ArrayAdapter.from_array(positioner.data, metadata=md) + + +def read_mda_scan(scan): + scan_md = dict( + dim=scan.dim, + number_detectors=scan.nd, + number_points_acquired=scan.curr_pt, + number_points_requested=scan.npts, + number_positioners=scan.np, + number_triggers=scan.nt, + PV=as_str(scan.name), + rank=scan.rank, + time=as_str(scan.time), # TODO: convert to timestamp (need TZ) + time_zone="US/Central (assumed since not in MDA file)", + ) + arrays = {} + for detector in scan.d: + k, v = read_mda_scan_detector(detector) + arrays[k] = v + for positioner in scan.p: + k, v = read_mda_scan_positioner(positioner) + arrays[k] = v + + for i, trigger in enumerate(scan.t, start=1): + # stored with scan metadata + v = {k: getattr(trigger, k) for k in "command number".split()} + v["EPICS_PV"] = as_str(trigger.name) + scan_md[f"T{i}"] = v + + return MapAdapter(arrays, metadata=scan_md, specs=[MDA_SCAN_SPECIFICATION]) + + +def read_mda(filename, **kwargs): + mda_obj = mda.readMDA( + str(filename), + useNumpy=True, + verbose=False, + showHelp=False, + ) + return MapAdapter( + {f"S{scan.rank}": read_mda_scan(scan) for scan in mda_obj[1:]}, + metadata=read_mda_header(mda_obj), + specs=[MDA_FILE_SPECIFICATION], + ) + + +def developer(): + import pathlib + + path = ( + pathlib.Path().home() + / "Documents" + / "projects" + / "NeXus" + / "exampledata" + / "APS" + / "scan2nexus" + ) + for filename in sorted(path.iterdir()): + if filename.name.endswith(EXTENSIONS[0]): + structure = read_mda(filename) + print(f"{filename.name=}") + print(f"{structure}") + + +if __name__ == "__main__": + developer() From d55f6d99dafcf7b27e2b980d8d2f53b39903e775 Mon Sep 17 00:00:00 2001 From: Rafael Vescovi Date: Mon, 21 Jul 2025 16:14:36 -0500 Subject: [PATCH 2/2] Add tiled integration files - docs/tiled/: Documentation for tiled integration - src/apsbits/demo_tiled/: Demo tiled server scripts and configuration - Provides example tiled server setup and management scripts --- docs/tiled/create.md | 326 ++++++++++++++++++ docs/tiled/documentation.md | 261 ++++++++++++++ docs/tiled/notes-2023-08-31.md | 58 ++++ src/apsbits/demo_tiled/scripts/in-screen.sh | 11 + .../demo_tiled/scripts/recreate_sampler.sh | 57 +++ src/apsbits/demo_tiled/scripts/start-tiled.sh | 23 ++ .../demo_tiled/scripts/tiled-manage.sh | 139 ++++++++ 7 files changed, 875 insertions(+) create mode 100644 docs/tiled/create.md create mode 100644 docs/tiled/documentation.md create mode 100644 docs/tiled/notes-2023-08-31.md create mode 100755 src/apsbits/demo_tiled/scripts/in-screen.sh create mode 100755 src/apsbits/demo_tiled/scripts/recreate_sampler.sh create mode 100755 src/apsbits/demo_tiled/scripts/start-tiled.sh create mode 100755 src/apsbits/demo_tiled/scripts/tiled-manage.sh diff --git a/docs/tiled/create.md b/docs/tiled/create.md new file mode 100644 index 00000000..7ae0874c --- /dev/null +++ b/docs/tiled/create.md @@ -0,0 +1,326 @@ +# Guide to Creating a `tiled` Server + +There are a few steps to create a `tiled` server for bluesky data. + +Jan Ilavsky has created an [article](https://github.com/jilavsky/SAXS_IgorCode/wiki/Reading-data-from-Tiled-server) +about reading data from such a server. + +CONTENTS + +- [Guide to Creating a `tiled` Server](#guide-to-creating-a-tiled-server) + - [Download the template](#download-the-template) + - [Create conda environment](#create-conda-environment) + - [Configure](#configure) + - [`config.yml`](#configyml) + - [databroker catalogs](#databroker-catalogs) + - [EPICS Area Detector data files](#epics-area-detector-data-files) + - [local data files](#local-data-files) + - [Run the server](#run-the-server) + - [Enable auto (re)start](#enable-auto-restart) + - [Clients](#clients) + +## Download the template + +The download steps must be done on a workstation that can reach the public +network. Use the same account that will be used to run the tiled server. + +The tiled server should run on a workstation that has access to the controls +subnet and any relevant filesystems with data to be served. + +```bash +cd your/projects/directory +git clone https://github.com/BCDA-APS/tiled-template ./tiled-server +cd ./tiled-server +``` + +TODO: What about changing the cloned repo origin? + +## Create conda environment + +```bash +conda env create --force -n tiled -f environment.yml --solver=libmamba +conda activate tiled +``` + +This may install a few hundred packages, including databroker v2+. + +
+Might seem slow... + +In a networked scenario like the APS, with many filesystems provided by NFS +exports and file backup & cache automation, processes that write many files to +NFS filesystems (such as creating a conda environment) may be very slow. It +could take 5-10 minutes to create this conda environment. Compare with the +procedures for creating a [conda environment for bluesky +operations](https://bcda-aps.github.io/bluesky_training/reference/_create_conda_env.html). +Many of the same advisories apply here, too. + +
+ +## Configure + +### `config.yml` + +Create your tiled configuration file from the template provided. + +```bash +cp config.yml.template config.yml +``` + +Keep in mind, YAML, like Python uses indentation as syntax. + +#### databroker catalogs + +Edit `config.yml` for your databroker catalog information: + +- `path`: name of this catalog (use this name from your bluesky sessions); can be found in: + - `bluesky/instrument/iconfig.yml` + - catalog name is at the end of line ~8: `DATABROKER_CATALOG: &databroker_catalog some_catalog_name` +- `uri` : address of your MongoDB catalog; in `mongodb://DB_SERVER.xray.aps.anl.gov:27017/45id_instrument-bluesky` replace: + - `DB_SERVER` with `db_host_name` (can be found in 2nd column of [APS list table](https://github.com/BCDA-APS/bluesky_training/wiki)) + - `45id_instrument` with `catalog_name` +- In line 4 `http://SERVER.xray.aps.anl.gov:8020/`: + - replace `SERVER` with the host name (computer running the tiled server) + - make sure the port number is consistent with the `./start-tiled.sh` script + +WARNING: consider whether you want this information publicly available or not (i.e. host tiled repo on aps gitlab or github) + +Repeat this block if you have more than one catalog to be served (such as +retired catalogs). A comment section of the template shows how to add addtional +catalogs. + +
+ +Sharp-eyed observers will note that the databroker configuration details specified for tiled are different than the ones they have been using with databroker v1.2. +The config for tiled uses the same info but in databroker v2 format. + +databroker v1.2 format + +```yaml + example: + args: + asset_registry_db: mongodb://mymongoserver.localdomain:27017/example + metadatastore_db: mongodb://mymongoserver.localdomain:27017/example + driver: bluesky-mongo-normalized-catalog +``` + +same content in tiled format + +```yaml + - path: example + tree: databroker.mongo_normalized:Tree.from_uri + args: + uri: mongodb://mymongoserver.localdomain:27017/example +``` + +Why the change? databroker is moving away from the intake library (the one that +reads the v1.2 format). `intake` seems to be slow to load (you see that when +importing databroker v1.2). New databroker v2 does not use intake. And is +faster to import. (Other improvements under the hood.) + +
+ +#### EPICS Area Detector data files + +If your files are written by EPICS area detector during bluesky runs, you do not +need to add file directories to your tiled server configuration if these +conditions are met: + +- Lightweight references to the file(s) and image(s) were written in databroker + (standard ophyd practice). +- Referenced files are available to the tiled server when their data is + requested by a client of the tiled server. + +
+Missing files... + +If a client requests data that comes from a referenced file and that file is not +available at the time of the request, the tiled server will return a *500 +Internal Server Error* to the client. For security reasons, a more detailed +answer is not provided to the tiled client. The tiled server console will +usually provide the detail that the file could not be found. + +
+ +#### local data files + +Skip this section if you are just getting started. + +*If* you want tiled to serve data files, the config file becomes longer. The +`config.yml` file has examples. Each file directory tree (including all its +subdirectories) is a separate entry in the `config.yml` file and a separate +SQLite file. + +Note: Very likely that details are missing in this section. Ask for help or +create an [issue](https://github.com/BCDA-APS/tiled-template/issues/new). + +
+Steps to add a directory tree + +Note: This is documentation is preliminary. + +For each directory tree, these steps: + +1. Identify a data file directory tree to be served by tiled. + 1. Create a new block in `config.yml` for the tree. + 1. Assign a name (like a catalog name) to identify the directory tree. +1. Recognize files by *mimetype*. + 1. Prepare Python code that recognizes new file types and assigns *mimetype* to each. + 1. Recognized by common file extension (such as `.mda` or `.xml`). + 1. Recognized by content analysis (such as NeXus, SPEC, or XML). + 1. Prepare Python tiled *adapter* code for each new *mimetype*. + 1. Add line(s) for each new *mimetype* to `config.yml`. +1. Create an SQLite catalog for the directory tree. + 1. Shell script `recreate_sampler.sh` + 1. `SQL_CATALOG=dev_sampler.sql`: name of SQLite file to be (re)created + 1. `FILE_DIR=./dev_sampler` : directory to be served + 1. Example (hypothetical) local directory + 1. Directory: `./dev_sampler` (does not exist in template here) + 1. Contains these types of file: MDA, NeXus, SPEC, images, XML, HDF4, text +1. Add SQLite file details to `config.yml` file: + + ```yaml + args: + uri: ./dev_sampler.sql + readable_storage: + - ./dev_sampler + ``` + +
+ +
+Details + +You specify data files by providing their directory (which includes all +subdirectories within). + +Files are recognized by +[*mimetype*](https://stackoverflow.com/questions/3828352/what-is-a-mime-type). +The configuration template has several examples. Here is an example for a SPEC +data file: + +```yaml + text/x-spec_data: spec_data:read_spec_data +``` + +The *mimetype* is `text/x-spec_data`. The adapter is the `read_spec_data()` +function in file `spec_data.py` (in the same directory as the `config.yml`). + +Custom *mimetype*s, such as `text/x-spec_data` are assigned in function +`detect_mimetype()` (in local file `custom.py`). This code identifies SPEC, +NeXus, and (non-NeXus) HDF5 files. + +Well-known file types, such as JPEG, TIFF, PNG, plain text, are recognized by +library functions called by the tiled server library code. + +For the SQLite file (at least at APS beamlines), keep in mind that NFS file +access is noticeably slower than local file access. It is recommended to store +the SQLite file on a local filesystem for the tiled server. + +
+ +## Run the server + +A bash shell script is available to run your tiled server. Take note of two important environment variables: + +- `HOST`: What client IP numbers will this server respond to? If `0.0.0.0`, the + server will respond to clients from any IP number. If `127.0.0.1`, the server + will only respond to clients on this workstation (localhost). +- `PORT`: What port will this server listen to? Your choice here. The default + choice here is arbitrary yet advised. Port 8000 is common but may be used by + some other local web server software. We choose port 8020 to avoid this + possibility. + +Once the `config.yml` and `start-tiled.sh` (and any configured SQLite) files are +prepared, start the tiled server for testing: + +
+$ ./start-tiled.sh
+
+ +Here is the output from my tiled server as it starts: + +```bash +Using configuration from /home/beams1/JEMIAN/Documents/projects/BCDA-APS/tiled-template/config.yml + + Tiled server is running in "public" mode, permitting open, anonymous access + for reading. Any data that is not specifically controlled with an access + policy will be visible to anyone who can connect to this server. + + + Navigate a web browser or connect a Tiled client to: + + http://0.0.0.0:8020?api_key=d8edc247909a0246b4e2dd8ca8d75443f87f2c5facd627b703d6635284e2f2fc + + + Because this server is public, the '?api_key=...' portion of + the URL is needed only for _writing_ data (if applicable). + + +INFO: Started server process [2033851] +INFO: Waiting for application startup. +OBJECT CACHE: Will use up to 1_190_568_960 bytes (15% of total physical RAM) +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:8020 (Press CTRL+C to quit) +``` + +Note: In this example, the `api_key` is randomly chosen by the server as it +starts. With this option for tiled server startup, a new key is generated each +time. A local installation should make a different choice, to provide its own +key to allow authorized clients to write data (as bluesky documents from a +RunEngine subscription). + +Enter the server URL (above: `http://0.0.0.0:8020`, not `https`) in a web +browser to test the server responds. Observe the server's console output each +time the web browser makes a new request. + +Press `^C` to quit the server. + +## Enable auto (re)start + +A bash shell script is available to help you manage the tiled server. It runs +the tiled server in a screen sessions (so the server does not quit when you +logout). The help command shows the commands available: + +
+$ ./tiled-manage.sh help
+Usage: ./tiled-manage.sh {start|stop|restart|checkup|status}
+
+ +For example, this linux command shows the server status on my workstation: + +```bash +./tiled-manage.sh status +# [2023-12-08T11:06:36-06:00 ./tiled-manage.sh] running fine, so it seems +``` + +Launch the server (for regular use): + +```bash +./tiled-manage.sh start +``` + +The `checkup` command may be used to (re)start the server. For example, to +enable automatic (re)start, add this line to your linux `cron` tasks. + +```cron +*/5 * * * * /full/path/to/your/tiled-server/tiled-manage.sh checkup 2>&1 > /dev/null +``` + +Linux command `crontab -e` will open an editor where you can paste this line. +The `tiled-manage.sh checkup` task will run every 5 minutes (9:10, 9:15, 9:20, +...). Within 5 minutes of a workstation reboot, the tiled server will be +started. + +## Clients + +Enter the server URL (above: `http://0.0.0.0:8020`, not `https`) in a web +browser to test the server responds. Observe the server's console output each +time the web browser makes a new request. + +You can use a web browser or find it more convenient to develop your own code +that makes requests using either URIs or Python `tiled.client` calls. + + +[`Gemviz`](https://bcda-aps.github.io/gemviz/), a Python Qt5 GUI program, is +being developed to browse and visualize data from your databroker catalogs. diff --git a/docs/tiled/documentation.md b/docs/tiled/documentation.md new file mode 100644 index 00000000..b3694a9d --- /dev/null +++ b/docs/tiled/documentation.md @@ -0,0 +1,261 @@ +# tiled + +APS local tiled data server template: databroker catalog + +- [tiled](#tiled) + - [Overview](#overview) + - [Startup](#startup) + - [Features](#features) + - [Additional file content served](#additional-file-content-served) + - [File Directories](#file-directories) + - [Indexing](#indexing) + - [Serve the catalog file](#serve-the-catalog-file) + - [Index the directory into the catalog file](#index-the-directory-into-the-catalog-file) + - [Custom file types](#custom-file-types) + - [Start tiled server with directory HDF5 files](#start-tiled-server-with-directory-hdf5-files) + - [Links](#links) + - [Install](#install) + - [Files](#files) + - [bluesky.yml](#blueskyyml) + +## Overview + +Run the *tiled* data server locally on workstation `SERVER`. Since this server +provides open access, it is only accessible within the APS firewall. + +- [x] databroker/MongoDB catalogs +- [x] file directories +- [ ] Authentication + +## Startup + +To start this tiled server (after configuring as described in the +[Install](#install} section), navigate to this directory and run the server +within a [screen](https://www.man7.org/linux/man-pages/man1/screen.1.html) +session: + +```bash +in-screen.sh +``` + +
+Tutorial: screen + +See also: https://www.hostinger.com/tutorials/how-to-install-and-use-linux-screen/ + +
+ +Then, use any web browser (within the APS firewall) to visit URL: +`http://SERVER:8000`. + +The web interface is a simple (simplistic yet informative) User Interface +demonstrating many features of the tiled server and also providing access to +online documentation. Visit the documentation to learn how to build your own +interface to tiled. + +### Features + +- serve data from Bluesky databroker catalogs +- (optional) serve data from user experiment file directory + +#### Additional file content served + +- [x] Identify NeXus/HDF5 files with arbitrary names. +- [x] Identify SPEC data files with arbitrary names and read them. +- [x] Read `.jpg` (and other image format) files. +- [x] Read the [synApps MDA format](https://github.com/epics-modules/sscan/blob/master/documentation/saveData_fileFormat.txt) ([Python support](https://github.com/EPICS-synApps/utils/blob/master/mdaPythonUtils/INSTALL.md)) +- [x] Write a custom data file identifier. +- [x] Write a custom data file loader. +- [x] Learn how to ignore files such as `.xml` (without startup comments). + +## File Directories + +Since tiled tag 0.1.0a104, serving a directory of files from tiled has become a +two-step process: + +1. Index the directory of files (into a SQLite file). +2. Serve the directory based on the index file. + +### Indexing + +Each tiled *tree* of a file directory, needs its own index. The index is a +local SQLite database file, (a.k.a., a *catalog*) that contains metadata +collected from each of the files and subdirectories for this tree. + +### Serve the catalog file + +Serve the catalog file. The name of this file can be anything (permissable by +the OS). To be consistent with the *tiled* documentation, we'll use +`catalog.db` for these examples. This is a one-time command, unless you wish to +remove any existing content from this SQL database. + +```bash +tiled catalog init catalog.db +``` + +### Index the directory into the catalog file + +Index the entire directory (and any subdirectories). This example walks through +the (local) `.dev_data/hdf` directory and indexes any files already recognized +by tiled. Also, it recognizes any files with suffixes `.nx5` and `.nexus.hdf5` +as HDF5. *tiled* already handles HDF5 as a file type, so no additional code is +required to parse and provide that content. + +```bash +tiled catalog register catalog.db \ + --verbose \ + --ext '.nx5=application/x-hdf5' \ + --ext '.nexus.hdf5=application/x-hdf5' \ + ./dev_data/hdf5 +``` + +#### Custom file types + +The `config.yml.template` has examples for custom file types. The command to +index changes. First, it is necessary to add the `*.py` files in this +directory, by prefixing the command with an environment definition for just this +command: `PYTHONPATH=. tiled catalog register ...` + +Next, add `--ext` options for each file suffix to be recognized. The +`--mimetype-hook` option identifies the local code to associate mimetypes with +any other unrecognized files. (For example, SPEC data files are text and may +not even have a common file suffix.) The `--adapter` lines define the local +custom code associated with each additional mimetype. + +Here's an example for the custom handlers in this repository. Note this example +uses the `./dev_data/` directory, so the `catalog.db` must first be +[recreated](#serve-the-catalog-file). + +```bash +PYTHONPATH=. \ + tiled catalog register \ + catalog.db \ + --verbose \ + --ext '.avif=image/avif' \ + --ext '.dat=text/x-spec_data' \ + --ext '.docx=application/octet-stream' \ + --ext '.DS_Store=text/plain' \ + --ext '.h5=application/x-hdf5' \ + --ext '.hdf=application/x-hdf5' \ + --ext '.mda=application/x-mda' \ + --ext '.nexus.hdf5=application/x-hdf5' \ + --ext '.nx5=application/x-hdf5' \ + --ext '.pptx=application/octet-stream' \ + --ext '.pyc=application/octet-stream' \ + --ext '.webp=image/webp' \ + --mimetype-hook 'custom:detect_mimetype' \ + --adapter 'application/json=ignore_data:read_ignore' \ + --adapter 'application/octet-stream=ignore_data:read_ignore' \ + --adapter 'application/x-mda=synApps_mda:read_mda' \ + --adapter 'application/xop+xml=ignore_data:read_ignore' \ + --adapter 'application/zip=ignore_data:read_ignore' \ + --adapter 'image/avif=ignore_data:read_ignore' \ + --adapter 'image/bmp=image_data:read_image' \ + --adapter 'image/gif=image_data:read_image' \ + --adapter 'image/jpeg=image_data:read_image' \ + --adapter 'image/png=image_data:read_image' \ + --adapter 'image/svg+xml=ignore_data:read_ignore' \ + --adapter 'image/tiff=image_data:read_image' \ + --adapter 'image/vnd.microsoft.icon=image_data:read_image' \ + --adapter 'image/webp=image_data:read_image' \ + --adapter 'image/x-ms-bmp=image_data:read_image' \ + --adapter 'text/markdown=ignore_data:read_ignore' \ + --adapter 'text/plain=ignore_data:read_ignore' \ + --adapter 'text/x-python=ignore_data:read_ignore' \ + --adapter 'text/x-spec_data=spec_data:read_spec_data' \ + --adapter 'text/xml=ignore_data:read_ignore' \ + ./dev_data +``` + +### Start tiled server with directory HDF5 files + +If there is only one catalog (this catalog of directories) to be served by +tiled, then start the server (with this `command.db` file and `./dev_data/hdf5/` +directory) from the command line, such as: + +```bash + tiled serve catalog catalog.db -r ./dev_data/hdf5/ --host 0.0.0.0 --public +``` + +To run a tiled server for multiple catalogs, use a `config.yml` file. To +configure tiled for this example directory of HDF5 files, add this to the +`config.yml` file: + +```yaml + - path: HDF5-files + tree: tiled.catalog:from_uri + args: + uri: ./catalog.db + readable_storage: + - ./dev_data/hdf5 +``` + +then start the *tiled* server with `./start-tiled.sh` or similar. + +## Links + +- +- +- `screen` tutorial: See also: https://www.hostinger.com/tutorials/how-to-install-and-use-linux-screen/ + +## Install + +1. Setup and activate a custom conda environment as directed + in [`environment.yml`](./environment.yml). + + Note: This step defines a `CONDA_PREFIX` environment variable in the bash shell. Used below. +2. tiled's configuration file: `config.yml`: + 1. Copy the template file `config.yml.template` to `config.yml` + 2. `path` is the name that will be seen by the tiled clients. + 3. `tree` should not be changed + 4. for databroker catalogs, `uri` is the address + of the mongodb catalog for this `path` + 5. for file directories, `directory` is the path to + the directory. Either absolute or relative to the + directory of this README.md file. + 6. Uncomment and edit the second catalog (`tree: databroker `...), + copy and edit if more catalogs are to be served. + 7. Uncomment and edit the file directory (`tree: files`) + if you wish tomake a file directory available. +3. Edit bash starter shell script file [`start-tiled.sh`](./start-tiled.sh) + 1. Override definition of `MY_DIR` at your choice. + 2. (optional) Activate the micromamba/conda environment (if not done + in step 1 above). You may need to change the definition of + `CONDA_ENV` which is the name of the conda environment to use. + 3. (optional) Change the `HOST` and `PORT` if needed. + 4. (optional) Remove the `--public` option if you want to require an + authentication token (shown on the console at startup of tiled). +4. Edit web interface to display additional columns: + 1. In the `$CONDA_PREFIX` directory, edit file + `share/tiled/ui/config/bluesky.yml` so it has the + content indicated by the [`bluesky.yml`](#blueskyyml) + below. + 2. Edit file `share/tiled/ui/configuration_manifest.yml` and + add a line at the bottom to include the `bluesky.yml` file: + + ```yml + - config/bluesky.yml + ``` + +## Files + +### bluesky.yml + +```yml +specs: + - spec: CatalogOfBlueskyRuns + columns: + - header: Bluesky Plan + select_metadata: start.plan_name + field: plan_name + - header: Scan ID + select_metadata: start.scan_id + field: scan_id + - header: Time + select_metadata: start.time + field: start_time + default_columns: + - plan_name + - scan_id + - start_time +``` diff --git a/docs/tiled/notes-2023-08-31.md b/docs/tiled/notes-2023-08-31.md new file mode 100644 index 00000000..c811082f --- /dev/null +++ b/docs/tiled/notes-2023-08-31.md @@ -0,0 +1,58 @@ +# NOTES with new tiled a105 + +terse notes collected from conversation 2023-08-31 + +```bash + tiled catalog init catalog.db + tiled catalog register catalog.db \ + --verbose \ + --ext '.nx5=application/x-hdf5' \ + --ext '.nexus.hdf5=application/x-hdf5' \ + ./dev_data/hdf5 + tiled serve catalog catalog.db -r ./dev_data/hdf5/ --host 0.0.0.0 --public +``` + + +```bash + tiled catalog init catalog.db + PYTHONPATH=. \ + tiled catalog register \ + catalog.db \ + --verbose \ + --ext '.avif=image/avif' \ + --ext '.dat=text/x-spec_data' \ + --ext '.docx=application/octet-stream' \ + --ext '.DS_Store=text/plain' \ + --ext '.h5=application/x-hdf5' \ + --ext '.hdf=application/x-hdf5' \ + --ext '.mda=application/x-mda' \ + --ext '.nexus.hdf5=application/x-hdf5' \ + --ext '.nx5=application/x-hdf5' \ + --ext '.pptx=application/octet-stream' \ + --ext '.pyc=application/octet-stream' \ + --ext '.webp=image/webp' \ + --mimetype-hook 'custom:detect_mimetype' \ + --adapter 'application/json=ignore_data:read_ignore' \ + --adapter 'application/octet-stream=ignore_data:read_ignore' \ + --adapter 'application/x-mda=synApps_mda:read_mda' \ + --adapter 'application/xop+xml=ignore_data:read_ignore' \ + --adapter 'application/zip=ignore_data:read_ignore' \ + --adapter 'image/avif=ignore_data:read_ignore' \ + --adapter 'image/bmp=image_data:read_image' \ + --adapter 'image/gif=image_data:read_image' \ + --adapter 'image/jpeg=image_data:read_image' \ + --adapter 'image/png=image_data:read_image' \ + --adapter 'image/svg+xml=ignore_data:read_ignore' \ + --adapter 'image/tiff=image_data:read_image' \ + --adapter 'image/vnd.microsoft.icon=image_data:read_image' \ + --adapter 'image/webp=image_data:read_image' \ + --adapter 'image/x-ms-bmp=image_data:read_image' \ + --adapter 'text/markdown=ignore_data:read_ignore' \ + --adapter 'text/plain=ignore_data:read_ignore' \ + --adapter 'text/x-python=ignore_data:read_ignore' \ + --adapter 'text/x-spec_data=spec_data:read_spec_data' \ + --adapter 'text/xml=ignore_data:read_ignore' \ + ./dev_data + tiled serve catalog catalog.db -r ./dev_data/ --host 0.0.0.0 --public +``` + diff --git a/src/apsbits/demo_tiled/scripts/in-screen.sh b/src/apsbits/demo_tiled/scripts/in-screen.sh new file mode 100755 index 00000000..6f14ab95 --- /dev/null +++ b/src/apsbits/demo_tiled/scripts/in-screen.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# run the tiled server in a screen session +SESSION_NAME=tiled_server +N_LINES=5000 + +screen \ + -dm \ + -S "${SESSION_NAME}" \ + -h "${N_LINES}" \ + ./start-tiled.sh diff --git a/src/apsbits/demo_tiled/scripts/recreate_sampler.sh b/src/apsbits/demo_tiled/scripts/recreate_sampler.sh new file mode 100755 index 00000000..c63d1974 --- /dev/null +++ b/src/apsbits/demo_tiled/scripts/recreate_sampler.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# re-create the SQLite catalog for the ./dev_sampler/ directory + +# ./recreate_sampler.sh 2>&1 | tee recreate.log + +SQL_CATALOG=dev_sampler.sql +FILE_DIR=./dev_sampler + +echo "Deleting '${SQL_CATALOG}', if it exists." +/bin/rm -f "${SQL_CATALOG}" + +echo "Creating '${SQL_CATALOG}' for directory '${FILE_DIR}'." +tiled catalog init "${SQL_CATALOG}" + +PYTHONPATH=. \ + tiled catalog register \ + "${SQL_CATALOG}" \ + -vvv \ + --keep-ext \ + --ext '.avif=image/avif' \ + --ext '.dat=text/x-spec_data' \ + --ext '.docx=application/octet-stream' \ + --ext '.DS_Store=text/plain' \ + --ext '.h5=application/x-hdf5' \ + --ext '.hdf=application/x-hdf5' \ + --ext '.mda=application/x-mda' \ + --ext '.nexus.hdf5=application/x-hdf5' \ + --ext '.nx5=application/x-hdf5' \ + --ext '.pptx=application/octet-stream' \ + --ext '.pyc=application/octet-stream' \ + --ext '.spc=text/x-spec_data' \ + --ext '.spe=text/x-spec_data' \ + --ext '.spec=text/x-spec_data' \ + --ext '.webp=image/webp' \ + --mimetype-hook 'custom:detect_mimetype' \ + --adapter 'application/json=ignore_data:read_ignore' \ + --adapter 'application/octet-stream=ignore_data:read_ignore' \ + --adapter 'application/x-mda=synApps_mda:read_mda' \ + --adapter 'application/xop+xml=ignore_data:read_ignore' \ + --adapter 'application/zip=ignore_data:read_ignore' \ + --adapter 'image/avif=ignore_data:read_ignore' \ + --adapter 'image/bmp=image_data:read_image' \ + --adapter 'image/gif=image_data:read_image' \ + --adapter 'image/jpeg=image_data:read_image' \ + --adapter 'image/png=image_data:read_image' \ + --adapter 'image/svg+xml=ignore_data:read_ignore' \ + --adapter 'image/tiff=image_data:read_image' \ + --adapter 'image/vnd.microsoft.icon=image_data:read_image' \ + --adapter 'image/webp=image_data:read_image' \ + --adapter 'image/x-ms-bmp=image_data:read_image' \ + --adapter 'text/markdown=ignore_data:read_ignore' \ + --adapter 'text/plain=ignore_data:read_ignore' \ + --adapter 'text/x-python=ignore_data:read_ignore' \ + --adapter 'text/x-spec_data=spec_data:read_spec_data' \ + --adapter 'text/xml=ignore_data:read_ignore' \ + "${FILE_DIR}" diff --git a/src/apsbits/demo_tiled/scripts/start-tiled.sh b/src/apsbits/demo_tiled/scripts/start-tiled.sh new file mode 100755 index 00000000..d4122f67 --- /dev/null +++ b/src/apsbits/demo_tiled/scripts/start-tiled.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# run the tiled server + +MY_DIR=$(realpath "$(dirname $0)") +LOG_FILE="${MY_DIR}/logfile.txt" +HOST=0.0.0.0 # access server by "localhost", hostname, or IP number +# HOST="${HOSTNAME}" # only access server by this exact name +PORT=8020 + +source ${CONDA_PREFIX}/etc/profile.d/conda.sh +CONDA_ENV=tiled +conda activate "${CONDA_ENV}" + + +# strace -fe openat,lstat \ + +tiled serve config \ + --port ${PORT} \ + --host ${HOST} \ + --public \ + "${MY_DIR}/config.yml" \ + 2>&1 | tee "${LOG_FILE}" diff --git a/src/apsbits/demo_tiled/scripts/tiled-manage.sh b/src/apsbits/demo_tiled/scripts/tiled-manage.sh new file mode 100755 index 00000000..f809bfea --- /dev/null +++ b/src/apsbits/demo_tiled/scripts/tiled-manage.sh @@ -0,0 +1,139 @@ +#!/bin/bash +# init file for tiled server +# +# chkconfig: - 98 98 +# description: tiled server +# +# processname: tiled_server + +SHELL_SCRIPT_NAME=${BASH_SOURCE:-${0}} + +PROJECT_DIR=$(dirname $(readlink -f "${SHELL_SCRIPT_NAME}")) +MANAGE="${PROJECT_DIR}/tiled-manage.sh" +LOGFILE="${PROJECT_DIR}/tiled-manage.log" +PIDFILE="${PROJECT_DIR}/tiled-manage.pid" +EXECUTABLE_SCRIPT="${PROJECT_DIR}/in-screen.sh" +STARTER_SCRIPT=start-tiled.sh +RETVAL=0 +SLEEP_DELAY=1.5 # wait for process, sometimes +TILED_CONDA_ENV=tiled + + +activate_conda(){ + if [ "${CONDA_EXE}" == "" ]; then + echo "Need CONDA_EXE defined to activate '${TILED_CONDA_ENV}' environment." + echo "That is defined by activating *any* conda environment." + exit 1 + fi + CONDA_ROOT=$(dirname $(dirname $(readlink -f "${CONDA_EXE}"))) + source "${CONDA_ROOT}/etc/profile.d/conda.sh" + conda activate "${TILED_CONDA_ENV}" +} + + +get_pid(){ + PID=$(/bin/cat "${PIDFILE}") + return $PID +} + + +function pid_is_running(){ + get_pid + if [ "${PID}" == "" ]; then + # no PID in the PIDFILE + RETVAL=1 + else + RESPONSE=$(ps -p ${PID} -o comm=) + if [ "${RESPONSE}" == "${STARTER_SCRIPT}" ]; then + # PID matches the tiled server profile + RETVAL=0 + else + # PID is not tiled server + RETVAL=1 + fi + fi + return "${RETVAL}" +} + + +start(){ + activate_conda + cd "${PROJECT_DIR}" + "${EXECUTABLE_SCRIPT}" 2>&1 >> "${LOGFILE}" & + sleep "${SLEEP_DELAY}" + PID=$(pidof -x ${STARTER_SCRIPT}) + /bin/echo "${PID}" > "${PIDFILE}" + /bin/echo \ + "# [$(/bin/date -Is) $0] started ${PID}: ${EXECUTABLE_SCRIPT}" \ + 2>&1 \ + >> "${LOGFILE}" & + sleep "${SLEEP_DELAY}" + tail -1 "${LOGFILE}" +} + + +stop(){ + get_pid + + if pid_is_running; then + /bin/echo "# [$(/bin/date -Is) $0] stopping ${PID}: ${EXECUTABLE_SCRIPT}" 2>&1 >> ${LOGFILE} & + kill "${PID}" + else + /bin/echo "# [$(/bin/date -Is) $0] not running ${PID}: ${EXECUTABLE_SCRIPT}" 2>&1 >> ${LOGFILE} & + fi + sleep "${SLEEP_DELAY}" + tail -1 "${LOGFILE}" + + /bin/cp -f /dev/null "${PIDFILE}" +} + + +restart(){ + stop + start +} + + +status(){ + if pid_is_running; then + echo "# [$(/bin/date -Is) $0] running fine, so it seems" + else + echo "# [$(/bin/date -Is) $0] could not identify running process ${PID}" + fi +} + + +checkup(){ + # 'crontab -e` to add entries for automated (re)start + #===================== + # call periodically (every 5 minutes) to see if tiled server is running + #===================== + # field allowed values + # ----- -------------- + # minute 0-59 + # hour 0-23 + # day of month 1-31 + # month 1-12 (or names, see below) + # day of week 0-7 (0 or 7 is Sun, or use names) + # + # */5 * * * * /home/beams/JEMIAN/Documents/projects/BCDA-APS/tiled-template/tiled-manage.sh checkup 2>&1 > /dev/null + + if pid_is_running; then + echo "# [$(/bin/date -Is) $0] running fine, so it seems" 2>&1 > /dev/null + else + echo "# [$(/bin/date -Is) $0] could not identify running process ${PID}, starting new process" 2>&1 >> "${LOGFILE}" + start + fi +} + + +case "$1" in + start) start ;; + stop) stop ;; + restart) restart ;; + checkup) checkup ;; + status) status ;; + *) + echo $"Usage: $0 {start|stop|restart|checkup|status}" + exit 1 +esac