From ac1f6c4de934fa928ccd727a3315f035d290702a Mon Sep 17 00:00:00 2001 From: "Christopher J. Wright" Date: Sun, 14 Jul 2019 21:26:46 -0400 Subject: [PATCH 1/9] move servers to using run router factories --- xpdan/startup/portable_db_server.py | 73 +++++++++++++++++------------ xpdan/startup/viz_server.py | 58 +++++++++++++++-------- 2 files changed, 82 insertions(+), 49 deletions(-) diff --git a/xpdan/startup/portable_db_server.py b/xpdan/startup/portable_db_server.py index 5cf314a..4eda7ec 100644 --- a/xpdan/startup/portable_db_server.py +++ b/xpdan/startup/portable_db_server.py @@ -38,34 +38,21 @@ """ -def run_server( - folder, - outbound_proxy_address=glbl_dict["outbound_proxy_address"], - prefix=None, - handlers=None, -): - """Start up the portable databroker server +def create_rr(folder, handlers=None): + """Create RunRouter for saving data to portable databrokers Parameters ---------- folder : str - The location where to save the portable databrokers - outbound_proxy_address : str, optional - The address and port of the zmq proxy. Defaults to - ``glbl_dict["outbound_proxy_address"]`` - prefix : bytes or list of bytes, optional - The Publisher channels to listen to. Defaults to - ``[b"an", b"raw"]`` - handlers : dict - The map between handler specs and handler classes, defaults to - the map used by the experimental databroker if possible + The folder to store the data into + handlers : dict, optional + The handlers for loading data + + Returns + ------- + RunRouter + """ - # TODO: convert to bytestrings if needed - # TODO: maybe separate this into different processes? - # TODO: support multiple locations for folders - if prefix is None: - prefix = [b"an", b"raw"] - d = RemoteDispatcher(outbound_proxy_address, prefix=prefix) portable_folder = folder portable_configs = {} for folder_name in ["an", "raw"]: @@ -116,22 +103,50 @@ def run_server( ) ).starsink(raw_broker.insert) - rr = RunRouter( + return RunRouter( [ lambda x: (lambda *nd: raw_source.emit(nd)) if x.get("analysis_stage", "") == "raw" else None ] + [ - lambda x: (lambda *nd: an_source.emit(nd)) - if x.get("analysis_stage", None) == "pdf" - else None, - lambda x: (lambda *nd: an_source.emit(nd)) - if x.get("analysis_stage", None) == "integration" + lambda x: (lambda *nd: raw_source.emit(nd)) + if x.get("analysis_stage", "") != "raw" else None, ] ) + +def run_server( + folder, + outbound_proxy_address=glbl_dict["outbound_proxy_address"], + prefix=None, + handlers=None, +): + """Start up the portable databroker server + + Parameters + ---------- + folder : str + The location where to save the portable databrokers + outbound_proxy_address : str, optional + The address and port of the zmq proxy. Defaults to + ``glbl_dict["outbound_proxy_address"]`` + prefix : bytes or list of bytes, optional + The Publisher channels to listen to. Defaults to + ``[b"an", b"raw"]`` + handlers : dict + The map between handler specs and handler classes, defaults to + the map used by the experimental databroker if possible + """ + # TODO: convert to bytestrings if needed + # TODO: maybe separate this into different processes? + # TODO: support multiple locations for folders + if prefix is None: + prefix = [b"an", b"raw"] + d = RemoteDispatcher(outbound_proxy_address, prefix=prefix) + rr = create_rr(folder, handlers=handlers) + d.subscribe(rr) print("Starting Portable DB Server") diff --git a/xpdan/startup/viz_server.py b/xpdan/startup/viz_server.py index a2958a8..478b411 100644 --- a/xpdan/startup/viz_server.py +++ b/xpdan/startup/viz_server.py @@ -27,6 +27,43 @@ def if_correct_start(callback, start_doc): return callback +def create_rr(handlers=None): + """Create RunRouter instance for visualizing data + + Parameters + ---------- + handlers : dict, optional + The handlers to use for loading externally stored data, if any + + Returns + ------- + RunRouter : + The RunRouter for data viz + """ + if handlers is None: + handlers = {} + func_l = [ + lambda x: if_correct_start( + LiveImage( + handler_reg=handlers, + cmap="viridis", + norm=SymLogNorm(1), + limit_func=lambda x: (np.nanmin(x), np.nanmax(x)), + ), + x, + ), + lambda x: LiveWaterfall(), + ] + if Live3DView: + func_l.append( + lambda x: Live3DView() if "tomo" in x["analysis_stage"] else None + ) + func_l.append( + lambda x: BestEffortCallback(table_enabled=False, overplot=False) + ) + return RunRouter(func_l) + + def run_server( handlers=None, prefix=None, @@ -57,26 +94,7 @@ def run_server( d = RemoteDispatcher(outbound_proxy_address, prefix=prefix) install_qt_kicker(loop=d.loop) - func_l = [ - lambda x: if_correct_start( - LiveImage( - handler_reg=handlers, - cmap="viridis", - norm=SymLogNorm(1), - limit_func=lambda x: (np.nanmin(x), np.nanmax(x)), - ), - x, - ), - lambda x: LiveWaterfall(), - ] - if Live3DView: - func_l.append( - lambda x: Live3DView() if "tomo" in x["analysis_stage"] else None - ) - func_l.append( - lambda x: BestEffortCallback(table_enabled=False, overplot=False) - ) - rr = RunRouter(func_l) + rr = create_rr(handlers) d.subscribe(rr) if save_folder: From e54e98221e35a69c0f496b618505c4a0254a7c5d Mon Sep 17 00:00:00 2001 From: "Christopher J. Wright" Date: Mon, 15 Jul 2019 15:00:11 -0400 Subject: [PATCH 2/9] fix bug which pushed all things to raw --- xpdan/startup/portable_db_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xpdan/startup/portable_db_server.py b/xpdan/startup/portable_db_server.py index 4eda7ec..3800d91 100644 --- a/xpdan/startup/portable_db_server.py +++ b/xpdan/startup/portable_db_server.py @@ -110,7 +110,7 @@ def create_rr(folder, handlers=None): else None ] + [ - lambda x: (lambda *nd: raw_source.emit(nd)) + lambda x: (lambda *nd: an_source.emit(nd)) if x.get("analysis_stage", "") != "raw" else None, ] From 2956fccdff4ec67775067ef9a717dcd3b8c6c2a6 Mon Sep 17 00:00:00 2001 From: "Christopher J. Wright" Date: Wed, 24 Jul 2019 12:58:26 -0400 Subject: [PATCH 3/9] new python --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a826e34..23fdf5b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ addons: - mongodb-org-server python: - - 3.6 + - 3.7 before_install: - "/sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1280x1024x16" From 03044dfc75a31ca179e10c2e42e39ad25b983370 Mon Sep 17 00:00:00 2001 From: "Christopher J. Wright" Date: Wed, 24 Jul 2019 12:59:30 -0400 Subject: [PATCH 4/9] remove fuzzydb --- score.yaml | 2 - xpdan/fuzzybroker.py | 320 -------------------------------- xpdan/tests/test_fuzzybroker.py | 75 -------- 3 files changed, 397 deletions(-) delete mode 100644 xpdan/fuzzybroker.py delete mode 100644 xpdan/tests/test_fuzzybroker.py diff --git a/score.yaml b/score.yaml index f7c8c42..50aedfb 100644 --- a/score.yaml +++ b/score.yaml @@ -25,8 +25,6 @@ run: default: {conda: ophyd} pyfai: default: {conda: pyfai} - pyxdameraulevenshtein: - default: {conda: pyxdameraulevenshtein} pyyaml: default: {conda: pyyaml} scikit-beam: diff --git a/xpdan/fuzzybroker.py b/xpdan/fuzzybroker.py deleted file mode 100644 index 9bc5158..0000000 --- a/xpdan/fuzzybroker.py +++ /dev/null @@ -1,320 +0,0 @@ -"""Enhanced databroker with fuzzy search options""" -############################################################################## -# -# xpdan by Billinge Group -# Simon J. L. Billinge sb2896@columbia.edu -# (c) 2016 trustees of Columbia University in the City of -# New York. -# All rights reserved -# -# File coded by: Christopher J. Wright -# -# See AUTHORS.txt for a list of people who contributed. -# See LICENSE.txt for license information. -# -############################################################################## - -from heapq import heapify, heappushpop -from pprint import pprint -from pyxdameraulevenshtein import \ - normalized_damerau_levenshtein_distance as ndld - -from databroker.broker import Broker -from .dev_utils import _timestampstr - - -class FuzzyBroker(Broker): - def fuzzy_search(self, keys, search_string, size=100): - """Fuzzy search a databroker for given keys - - Parameters - ---------- - keys: list of str - The list of strings to be accessed - search_string: str - The string to be searched for - size: int or 'all', optional - The number of results to be returned, if 'all' all are returned. - Defaults to 100 results - - Returns - ------- - list: - A list - - Notes - ------ - This search can take a long time as they turn over the entire - databroker searching for the ``search_string`` please make a point to - filter the databroker to shorten the total search time - - """ - return fuzzy_search(self, keys, search_string, size) - - def super_fuzzy_search(self, search_string, size=100): - """Fuzzy search a databroker - - Parameters - ---------- - search_string: str - The string to be searched for - size: int, optional - The number of results to be returned. - Defaults to 100 results - - Returns - ------- - list: - A list of headers which contain close matches - - Notes - ------ - This search can take a long time as they turn over the entire - databroker (and all of the dictionaries inside) searching for the - ``search_string`` please make a point to filter the databroker to - shorten the total search time. - - """ - return super_fuzzy_search(self, search_string, size) - - def beamtime_dates(self, **kwargs): - """Get info for each beamtime - - Parameters - ---------- - keys: iterable of str - The keys to be included in the return - beamtime_key: str - The key for the unique beamtime key - print_results: bool - If true prints the information - - Returns - ------- - list of dicts: - The list of beamtimes and their associated information - """ - return beamtime_dates(self, **kwargs) - - def fuzzy_set_search(self, key, search_string, size=100): - """Return the most similar set of values to the search string. - - Parameters - ---------- - key: list of str - The list of strings to be accessed - search_string: str - The string to be searched for - size: int, optional - The number of results to be returned. - Defaults to 100 results - - Returns - ------- - list: - A list of headers which are close to the query. The queried value - will be unique in the list so a search for piLast='Alice' will - return only one header with Alice as the PI. - - Examples - -------- - >>> db = Broker(...) # Contains runs from Bob, Alice, Bob, and Eve - >>> fuzzy_set_search(db, 'bt_piLast', 'Bob') - ['Bob', 'Alice', 'Eve'] - """ - return fuzzy_set_search(self, key, search_string, size) - - -def _get_from_dict(data_dict, map_list): - """Get a value from a nested dictionary, given a list of keys - - Parameters - ---------- - data_dict: dict - The dictionary to be queried - map_list: list of str - A list of strings, each string is one level lower than the previous - - Returns - ------- - object: - The the value from the dict - - """ - for k in map_list: - data_dict = data_dict[k] - return data_dict - - -def _nested_dict_values(d): - """Yield all string values inside a nested dictionary - - Parameters - ---------- - d: dict - The dictionary to be unpacked - - Yields - ------- - str: - The string value inside the dictionary - - """ - for v in d.values(): - if isinstance(v, dict): - yield from _nested_dict_values(v) - else: - if isinstance(v, str): - yield v - else: - yield None - - -def fuzzy_search(db, keys, search_string, size=100): - """Fuzzy search a databroker for given keys - - Parameters - ---------- - db: databroker.DataBroker instance - The databroker to be searched - keys: list of str - The list of strings to be accessed - search_string: str - The string to be searched for - size: int or 'all', optional - The number of results to be returned, if 'all' all are returned. - Defaults to 100 results - - Returns - ------- - list: - A list - - """ - heap = [(-1, -1, -1)] * size # ndld can't return less than 0 - heapify(heap) - if isinstance(keys, list): - for h in db(): - # prioritize recent documents - heappushpop(heap, (1. - ndld(_get_from_dict(h['start'], keys), - search_string), - h['start']['time'] * -1, h)) - else: - for h in db(): - heappushpop(heap, (1. - ndld(h['start'][keys], search_string), - h['start']['time'] * -1, h)) - heap.sort() - heap.reverse() - return [g[-1] for g in heap if g[0] >= 0.] - - -def super_fuzzy_search(db, search_string, size=100): - """Fuzzy search a databroker - - Parameters - ---------- - db: databroker.DataBroker instance - The databroker to be searched - search_string: str - The string to be searched for - size: int, optional - The number of results to be returned. - Defaults to 100 results - - Returns - ------- - list: - A list - - """ - heap = [(-1, -1, -1)] * size # ndld can't return less than 0 - heapify(heap) - for h in db(): - internal_scores = [1. - ndld(v, search_string) for v in - _nested_dict_values(h['start']) if v is not None] - heappushpop(heap, (max(internal_scores), h['start']['time'] * -1, h)) - heap.sort() - heap.reverse() - return [g[-1] for g in heap if g[0] != -1] - - -def beamtime_dates(db, keys=('beamtime_uid', 'bt_safN', - 'facility', 'beamline'), - beamtime_key='beamtime_uid', - print_results=True): - """Get info for each beamtime - - Parameters - ---------- - db: databroker instance - The databroker to be searched - keys: iterable of str - The keys to be included in the return - beamtime_key: str - The key for the unique beamtime key - print_results: bool - If true prints the information - - Returns - ------- - list of dicts: - The list of beamtimes and their associated information - """ - hdrs = db() - bts = set([h['start'][beamtime_key] for h in hdrs]) - returns = [] - for s in bts: - hdrs = db(**{beamtime_key: s}) - start_hdr = next(iter(hdrs)) - # XXX: this is bad but I have no other way to get the latest element - # from results - for hdr in hdrs: - stop_hdr = hdr - for i, hdr in enumerate(hdrs): - if i == 0: - start_hdr = hdr - pass - stop_hdr = hdr - info = {k: start_hdr[k] for k in keys if k in start_hdr.keys()} - info.update({'start_time': _timestampstr(start_hdr['start']['time']), - 'stop_time': _timestampstr(stop_hdr['start']['time'])}) - returns.append(info) - if print_results: - pprint(returns) - return returns - - -def fuzzy_set_search(db, key, search_string, size=100): - """Return the most similar set of values to the search string. - - Parameters - ---------- - db: databroker.DataBroker instance - The databroker to be searched - key: list of str - The list of strings to be accessed - search_string: str - The string to be searched for - size: int, optional - The number of results to be returned. - Defaults to 100 results - - Returns - ------- - list: - A list - - Examples - -------- - >>> db = Broker(...) # Contains runs from Bob, Alice, Bob, and Eve - >>> fuzzy_set_search(db, 'bt_piLast', 'Bob') - ['Bob', 'Alice', 'Eve'] - """ - heap = [(-1, -1)] * size # ndld can't return less than 0 - heapify(heap) - values = set([h['start'][key] for h in db()]) - for v in values: - heappushpop(heap, (1. - ndld(v, search_string), v)) - heap.sort() - heap.reverse() - return [g[-1] for g in heap if g[0] >= 0.] diff --git a/xpdan/tests/test_fuzzybroker.py b/xpdan/tests/test_fuzzybroker.py deleted file mode 100644 index b2ceed2..0000000 --- a/xpdan/tests/test_fuzzybroker.py +++ /dev/null @@ -1,75 +0,0 @@ -############################################################################## -# -# xpdan by Billinge Group -# Simon J. L. Billinge sb2896@columbia.edu -# (c) 2016 trustees of Columbia University in the City of -# New York. -# All rights reserved -# -# File coded by: Christopher J. Wright -# -# See AUTHORS.txt for a list of people who contributed. -# See LICENSE.txt for license information. -# -############################################################################## -from xpdan.fuzzybroker import (fuzzy_search, super_fuzzy_search, - beamtime_dates, fuzzy_set_search) -import pytest - - -@pytest.mark.parametrize(('search_str', 'target_str'), - [('chris', 'chris'), ('christ', 'chris'), - ('chry', 'chris'), ('tim', 'tim')]) -def test_fuzzy_searches(exp_db, search_str, target_str): - search_result = fuzzy_search(exp_db, 'pi_name', search_str) - assert search_result[0]['start']['pi_name'] == target_str - - -@pytest.mark.parametrize(('search_str', 'target_str'), - [('chris', 'chris'), ('christ', 'chris'), - ('chry', 'chris'), ('tim', 'tim')]) -def test_super_fuzzy_search(exp_db, search_str, target_str): - search_result = super_fuzzy_search(exp_db, search_str) - assert search_result[0]['start']['pi_name'] == target_str - - -def test_beamtime_dates_smoke(exp_db): - beamtime_dates(exp_db) - - -@pytest.mark.parametrize(('search_str', 'target_str'), - [('chris', 'chris'), ('christ', 'chris'), - ('chry', 'chris'), ('tim', 'tim')]) -def test_fuzzy_set_search(exp_db, search_str, target_str): - res = fuzzy_set_search(exp_db, 'pi_name', search_str) - assert res[0] == target_str - assert len(res) == 2 - - -@pytest.mark.parametrize(('search_str', 'target_str'), - [('chris', 'chris'), ('christ', 'chris'), - ('chry', 'chris'), ('tim', 'tim')]) -def test_class_fuzzy_searches(fuzzdb, search_str, target_str): - search_result = fuzzdb.fuzzy_search('pi_name', search_str) - assert search_result[0]['start']['pi_name'] == target_str - - -@pytest.mark.parametrize(('search_str', 'target_str'), - [('chris', 'chris'), ('christ', 'chris'), - ('chry', 'chris'), ('tim', 'tim')]) -def test_class_super_fuzzy_search(fuzzdb, search_str, target_str): - search_result = fuzzdb.super_fuzzy_search(search_str) - assert search_result[0]['start']['pi_name'] == target_str - - -def test_class_beamtime_dates_smoke(fuzzdb): - fuzzdb.beamtime_dates() - - -@pytest.mark.parametrize(('search_str', 'target_str'), - [('chris', 'chris'), ('christ', 'chris'), - ('chry', 'chris'), ('tim', 'tim')]) -def test_fuzzy_class_set_search(fuzzdb, search_str, target_str): - res = fuzzdb.fuzzy_set_search('pi_name', search_str) - assert res[0] == target_str - assert len(res) == 2 From f325822d5a008d28fa929b0f44fe709dcd2e0213 Mon Sep 17 00:00:00 2001 From: "Christopher J. Wright" Date: Wed, 24 Jul 2019 13:00:32 -0400 Subject: [PATCH 5/9] travis doesn't have 3.7 --- .travis.yml | 2 +- score.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 23fdf5b..a826e34 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ addons: - mongodb-org-server python: - - 3.7 + - 3.6 before_install: - "/sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1280x1024x16" diff --git a/score.yaml b/score.yaml index 50aedfb..959cf05 100644 --- a/score.yaml +++ b/score.yaml @@ -11,7 +11,7 @@ build: conda: setuptools python: default: - conda: python==$TRAVIS_PYTHON_VERSION + conda: python==3.7 run: bluesky: default: {conda: bluesky} From 91d7b0d2de83ed983414593d7151dadaafeecfac Mon Sep 17 00:00:00 2001 From: "Christopher J. Wright" Date: Wed, 24 Jul 2019 19:01:19 -0400 Subject: [PATCH 6/9] remove mayavi --- score.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/score.yaml b/score.yaml index 959cf05..c7de4c2 100644 --- a/score.yaml +++ b/score.yaml @@ -48,8 +48,6 @@ run: default: {conda: xray-vision} rapidz: default: {conda: rapidz} - mayavi: - default: {conda: mayavi} test: pytest: default: From 58c4c9f6fb53b4f544f1156f7bb65c1109fe7f46 Mon Sep 17 00:00:00 2001 From: "Christopher J. Wright" Date: Wed, 24 Jul 2019 23:15:47 -0400 Subject: [PATCH 7/9] any python --- score.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/score.yaml b/score.yaml index c7de4c2..84f5d4e 100644 --- a/score.yaml +++ b/score.yaml @@ -11,7 +11,7 @@ build: conda: setuptools python: default: - conda: python==3.7 + conda: python run: bluesky: default: {conda: bluesky} From a768393ccae912326460a985179f77a9d265e3bf Mon Sep 17 00:00:00 2001 From: "Christopher J. Wright" Date: Thu, 25 Jul 2019 08:24:34 -0400 Subject: [PATCH 8/9] no fuzzy --- xpdan/tests/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xpdan/tests/conftest.py b/xpdan/tests/conftest.py index 780ddc3..28fb500 100644 --- a/xpdan/tests/conftest.py +++ b/xpdan/tests/conftest.py @@ -35,7 +35,6 @@ ) from skbeam.io.fit2d import fit2d_save -from xpdan.fuzzybroker import FuzzyBroker from .utils import insert_imgs import matplotlib.pyplot as plt From c5ded4e5c9e2c02bb16756088e6edd93dcfb1f50 Mon Sep 17 00:00:00 2001 From: "Christopher J. Wright" Date: Thu, 25 Jul 2019 11:00:16 -0400 Subject: [PATCH 9/9] install pip --- score.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/score.yaml b/score.yaml index 84f5d4e..2b05279 100644 --- a/score.yaml +++ b/score.yaml @@ -8,7 +8,7 @@ install: pip install . build: pip: default: - conda: setuptools + conda: pip python: default: conda: python