diff --git a/README.md b/README.md
index 917e81c6..d5c4a5ad 100644
--- a/README.md
+++ b/README.md
@@ -314,6 +314,11 @@ First, if you have a custom configuration for your own setup, set the proper var
export REMOTE_OBSERVATORY_CONFIG=backyard_config
```
+## Run the config server
+```console
+PYTHONPATH=. python ./config/cli.py run --config-file ./conf_files/config.yaml
+```
+
## If you want to try the software with simulators:
```console
./apps/launch_indi_simu.sh
diff --git a/conf_files/config.yaml b/conf_files/config.yaml
index 6f53f9c3..d293cb66 100644
--- a/conf_files/config.yaml
+++ b/conf_files/config.yaml
@@ -15,10 +15,9 @@ observatory:
180 : 0
270 : 0
twilight_horizon: -18 # Degrees
- timezone: Europe/Paris
# Paris is in the Central European Time Zone ( CET ) is 1 hours ahead of
# Greenwich Mean Time ( GMT+1 ).
- gmt_offset: 60 # Minutes
+# gmt_offset: 60 # Minutes
# webcam:
# rtsp_url: rtsp://user:password@192.168.0.16
scope_controller:
diff --git a/config/__init__.py b/config/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/config/cli.py b/config/cli.py
new file mode 100644
index 00000000..72c2bb17
--- /dev/null
+++ b/config/cli.py
@@ -0,0 +1,156 @@
+import time
+
+import click
+from loguru import logger
+
+from config import (server)
+from config.client import get_config
+from config.client import server_is_running
+from config.client import set_config
+
+import logging
+logging.basicConfig(level=logging.DEBUG, format='%(asctime)s;%(levelname)s:%(message)s')
+logger = logging.getLogger(__name__)
+
+
+@click.group()
+@click.option('--verbose/--no-verbose',
+ envvar='DEBUG',
+ help='Turn on logger for utils, default False')
+@click.option('--host',
+ default=None,
+ envvar='CONFIG_HOST',
+ help='The config server IP address or host name. First'
+ 'checks cli argument, then CONFIG_HOST, then localhost.')
+@click.option('--port',
+ default=None,
+ envvar='CONFIG_PORT',
+ help='The config server port. First checks cli argument, '
+ 'then CONFIG_PORT, then 6563')
+@click.pass_context
+def config_server_cli(context, host='localhost', port=6563, verbose=False):
+ context.ensure_object(dict)
+ context.obj['host'] = host
+ context.obj['port'] = port
+
+ # if verbose:
+ # logger.enable('')
+
+
+@click.command('run')
+@click.option('--config-file',
+ default=None,
+ envvar='CONFIG_FILE',
+ help='The yaml config file to load.'
+ )
+@click.option('--load-local/--no-load-local',
+ default=True,
+ help='Use the local config files when loading, default True.')
+@click.option('--save-local/--no-save-local',
+ default=True,
+ help='If the set values should be saved to the local file, default True.')
+@click.option('--heartbeat',
+ default=2,
+ help='Heartbeat interval, default 2 seconds.')
+@click.pass_context
+def run(context, config_file=None, save_local=True, load_local=False, heartbeat=2):
+ """Runs the config server with command line options.
+
+ This function is installed as an entry_point for the module, accessible
+ at `config-server`.
+ """
+ host = context.obj.get('host')
+ port = context.obj.get('port')
+ logger.debug("About to run config server")
+ server_process = server.config_server(
+ config_file,
+ host=host,
+ port=port,
+ load_local=load_local,
+ save_local=save_local,
+ auto_start=False
+ )
+
+ try:
+ print(f'Starting config server. Ctrl-c to stop')
+ server_process.start()
+ print(f'Config server started on server_process.pid={server_process.pid!r}. '
+ f'Set "config_server.running=False" or Ctrl-c to stop')
+
+ # Loop until config told to stop.
+ while server_is_running(host=host, port=port):
+ time.sleep(heartbeat)
+
+ server_process.terminate()
+ server_process.join(30)
+ except KeyboardInterrupt:
+ logger.info(f'Config server interrupted, shutting down {server_process.pid}')
+ server_process.terminate()
+ except Exception as e: # pragma: no cover
+ logger.error(f'Unable to start config server {e!r}')
+
+
+@click.command('stop')
+@click.pass_context
+def stop(context):
+ """Stops the config server by setting a flag in the server itself."""
+ host = context.obj.get('host')
+ port = context.obj.get('port')
+ logger.info(f'Shutting down config server on {host}:{port}')
+ set_config('config_server.running', False, host=host, port=port)
+
+
+@click.command('get')
+@click.argument('key', nargs=-1)
+@click.option('--default', help='The default to return if not key is found, default None')
+@click.option('--parse/--no-parse',
+ default=True,
+ help='If results should be parsed into object, default True.')
+@click.pass_context
+def config_getter(context, key, parse=True, default=None):
+ """Get an item from the config server by key name, using dotted notation (e.g. 'location.elevation')
+
+ If no key is given, returns the entire config.
+ """
+ host = context.obj.get('host')
+ port = context.obj.get('port')
+ try:
+ # The nargs=-1 makes this a tuple so we get first entry.
+ key = key[0]
+ except IndexError:
+ key = None
+ logger.debug(f'Getting config key={key!r}')
+ try:
+ config_entry = get_config(key=key, host=host, port=port, parse=parse, default=default)
+ except Exception as e:
+ logger.error(f'Error while trying to get config: {e!r}')
+ click.secho(f'Error while trying to get config: {e!r}', fg='red')
+ else:
+ logger.debug(f'Config server response: config_entry={config_entry!r}')
+ click.echo(config_entry)
+
+
+@click.command('set')
+@click.argument('key')
+@click.argument('new_value')
+@click.option('--parse/--no-parse',
+ default=True,
+ help='If results should be parsed into object.')
+@click.pass_context
+def config_setter(context, key, new_value, parse=True):
+ """Set an item in the config server. """
+ host = context.obj.get('host')
+ port = context.obj.get('port')
+
+ logger.debug(f'Setting config key={key!r} new_value={new_value!r} on {host}:{port}')
+ config_entry = set_config(key, new_value, host=host, port=port, parse=parse)
+ click.echo(config_entry)
+
+
+config_server_cli.add_command(run)
+config_server_cli.add_command(stop)
+config_server_cli.add_command(config_setter)
+config_server_cli.add_command(config_getter)
+
+if __name__ == '__main__':
+ config_server_cli() # This is where click takes control
diff --git a/config/client.py b/config/client.py
new file mode 100644
index 00000000..0a943b85
--- /dev/null
+++ b/config/client.py
@@ -0,0 +1,202 @@
+import os
+
+import requests
+from loguru import logger
+from requests.exceptions import ConnectionError
+
+from utils.error import InvalidConfig
+from utils.serializers import from_json, to_json
+
+import logging
+logging.basicConfig(level=logging.DEBUG, format='%(asctime)s;%(levelname)s:%(message)s')
+logger = logging.getLogger(__name__)
+
+
+def server_is_running(*args, **kwargs): # pragma: no cover
+ """Thin-wrapper to check server."""
+ try:
+ return get_config(endpoint='heartbeat', verbose=False, *args, **kwargs)
+ except Exception as e:
+ logger.warning(f'server_is_running error (ignore if just starting server): {e!r}')
+ return False
+
+
+def get_config(key=None,
+ default=None,
+ host=None,
+ port=None,
+ endpoint='get-config',
+ parse=True,
+ verbose=False
+ ):
+ """Get a config item from the config server.
+
+ Return the config entry for the given ``key``. If ``key=None`` (default), return
+ the entire config.
+
+ Nested keys can be specified as a string, as per `scalpl `_.
+
+ Examples:
+
+ .. doctest::
+
+ >>> get_config(key='name')
+ 'Testing PANOPTES Unit'
+
+ >>> get_config(key='location.horizon')
+
+
+ >>> # With no parsing, the raw string (including quotes) is returned.
+ >>> get_config(key='location.horizon', parse=False)
+ '"30 deg"'
+ >>> get_config(key='cameras.devices[1].model')
+ 'canon_gphoto2'
+
+ >>> # Returns `None` if key is not found.
+ >>> foobar = get_config(key='foobar')
+ >>> foobar is None
+ True
+
+ >>> # But you can supply a default.
+ >>> get_config(key='foobar', default='baz')
+ 'baz'
+
+ >>> # key and default are first two parameters.
+ >>> get_config('foobar', 'baz')
+ 'baz'
+
+ >>> # Can use Quantities as well.
+ >>> from astropy import units as u
+ >>> get_config('foobar', 42 * u.meter)
+
+
+
+ Notes:
+ By default all calls to this function will log at the `trace` level because
+ there are some calls (e.g. during POCS operation) that will be quite noisy.
+
+ Setting `verbose=True` changes those to `debug` log levels for an individual
+ call.
+
+ Args:
+ key (str): The key to update, see Examples in :func:`get_config` for details.
+ default (str, optional): The config server port, defaults to 6563.
+ host (str, optional): The config server host. First checks for CONFIG_HOST
+ env var, defaults to 'localhost'.
+ port (str or int, optional): The config server port. First checks for CONFIG_HOST
+ env var, defaults to 6563.
+ endpoint (str, optional): The relative url endpoint to use for getting
+ the config items, default 'get-config'. See `server_is_running()`
+ for example of usage.
+ parse (bool, optional): If response should be parsed by
+ :func:`serializers.from_json`, default True.
+ verbose (bool, optional): Determines the output log level, defaults to
+ True (i.e. `debug` log level). See notes for details.
+ Returns:
+ dict: The corresponding config entry.
+
+ Raises:
+ Exception: Raised if the config server is not available.
+ """
+ log_level = 'DEBUG' if verbose else 'TRACE'
+
+ host = host or os.getenv('CONFIG_HOST', 'localhost')
+ port = port or os.getenv('CONFIG_PORT', 6563)
+
+ url = f'http://{host}:{port}/{endpoint}'
+
+ config_entry = default
+
+ try:
+ logger.debug(f'Calling get_config on url={url!r} with key={key!r}')
+ response = requests.post(url, json={'key': key, 'verbose': verbose})
+ if not response.ok: # pragma: no cover
+ raise InvalidConfig(f'Config server returned invalid JSON: {response.content!r}')
+ except ConnectionError:
+ logger.debug('Bad connection to config-server. Check to make sure it is running.')
+ except Exception as e: # pragma: no cover
+ logger.warning(f'Problem with get_config: {e!r}')
+ else:
+ response_text = response.text.strip()
+ logger.debug(f'Decoded response_text={response_text!r}')
+ if response_text != 'null':
+ logger.debug(f'Received config key={key!r} response_text={response_text!r}')
+ if parse:
+ logger.debug(f'Parsing config results: response_text={response_text!r}')
+ config_entry = from_json(response_text)
+ else:
+ config_entry = response_text
+
+ if config_entry is None:
+ logger.debug(f'No config entry found, returning default={default!r}')
+ config_entry = default
+
+ logger.debug(f'Config key={key!r}: config_entry={config_entry!r}')
+ return config_entry
+
+
+def set_config(key, new_value, host=None, port=None, parse=True):
+ """Set config item in config server.
+
+ Given a `key` entry, update the config to match. The `key` is a dot accessible
+ string, as given by `scalpl `_. See Examples in
+ :func:`get_config` for details.
+
+ Examples:
+
+ .. doctest::
+
+ >>> from astropy import units as u
+
+ >>> # Can use astropy units.
+ >>> set_config('location.horizon', 35 * u.degree)
+ {'location.horizon': }
+
+ >>> get_config(key='location.horizon')
+
+
+ >>> # String equivalent works for 'deg', 'm', 's'.
+ >>> set_config('location.horizon', '30 deg')
+ {'location.horizon': }
+
+ Args:
+ key (str): The key to update, see Examples in :func:`get_config` for details.
+ new_value (scalar|object): The new value for the key, can be any serializable object.
+ host (str, optional): The config server host. First checks for CONFIG_HOST
+ env var, defaults to 'localhost'.
+ port (str or int, optional): The config server port. First checks for CONFIG_HOST
+ env var, defaults to 6563.
+ parse (bool, optional): If response should be parsed by
+ :func:`serializers.from_json`, default True.
+
+ Returns:
+ dict: The updated config entry.
+
+ Raises:
+ Exception: Raised if the config server is not available.
+ """
+ host = host or os.getenv('CONFIG_HOST', 'localhost')
+ port = port or os.getenv('CONFIG_PORT', 6563)
+ url = f'http://{host}:{port}/set-config'
+
+ json_str = to_json({key: new_value})
+
+ config_entry = None
+ try:
+ # We use our own serializer so pass as `data` instead of `json`.
+ logger.debug(f'Calling set_config on url={url!r}')
+ response = requests.post(url,
+ data=json_str,
+ headers={'Content-Type': 'application/json'}
+ )
+ if not response.ok: # pragma: no cover
+ raise Exception(f'Cannot access config server: {response.text}')
+ except Exception as e:
+ logger.warning(f'Problem with set_config: {e!r}')
+ else:
+ if parse:
+ config_entry = from_json(response.content.decode('utf8'))
+ else:
+ config_entry = response.json()
+
+ return config_entry
diff --git a/config/helpers.py b/config/helpers.py
new file mode 100644
index 00000000..7a91c5f3
--- /dev/null
+++ b/config/helpers.py
@@ -0,0 +1,195 @@
+from contextlib import suppress
+from pathlib import Path
+from typing import Dict, List, Union
+
+from loguru import logger
+
+from utils import error
+from utils.serializers import from_yaml, to_yaml
+from utils import listify
+
+
+def load_config(config_files: Union[Path, List] = None, parse: bool = True,
+ load_local: bool = True):
+ """Load configuration information.
+
+ .. note::
+
+ This function is used by the config server and normal config usage should
+ be via a running config server.
+
+ This function supports loading of a number of different files. If no options
+ are passed to ``config_files`` then the default ``$CONFIG_FILE``
+ will be loaded.
+
+ ``config_files`` is a list and loaded in order, so the second entry will overwrite
+ any values specified by similarly named keys in the first entry.
+
+ ``config_files`` should be specified by an absolute path, which can exist anywhere
+ on the filesystem.
+
+ Local versions of files can override built-in versions and are automatically loaded if
+ they exist alongside the specified config path. Local files have a ``<>_local.yaml`` name, where
+ ``<>`` is the built-in file.
+
+ Given the following path:
+
+ ::
+
+ /path/to/dir
+ |- my_conf.yaml
+ |- my_conf_local.yaml
+
+ You can do a ``load_config('/path/to/dir/my_conf.yaml')`` and both versions of the file will
+ be loaded, with the values in the local file overriding the non-local. Typically the local
+ file would also be ignored by ``git``, etc.
+
+ For example, the ``config.server.config_server`` will always save values to
+ a local version of the file so the default settings can always be recovered if necessary.
+
+ Local files can be ignored (mostly for testing purposes or for recovering default values)
+ with the ``load_local=False`` parameter.
+
+ Args:
+ config_files (list, optional): A list of files to load as config,
+ see Notes for details of how to specify files.
+ parse (bool, optional): If the config file should attempt to create
+ objects such as dates, astropy units, etc.
+ load_local (bool, optional): If local files should be used, see
+ Notes for details.
+
+ Returns:
+ dict: A dictionary of config items.
+ """
+ config = dict()
+
+ config_files = listify(config_files)
+ logger.debug(f'Loading config files: config_files={config_files!r}')
+ for config_file in config_files:
+ config_file = Path(config_file)
+ try:
+ logger.debug(f'Adding config_file={config_file!r} to config dict')
+ _add_to_conf(config, config_file, parse=parse)
+ except Exception as e: # pragma: no cover
+ logger.warning(f"Problem with config_file={config_file!r}, skipping. {e!r}")
+
+ # Load local version of config
+ if load_local:
+ local_version = config_file.parent / Path(config_file.stem + '_local.yaml')
+ if local_version.exists():
+ try:
+ _add_to_conf(config, local_version, parse=parse)
+ except Exception as e: # pragma: no cover
+ logger.warning(f"Problem with local_version={local_version!r}, skipping: {e!r}")
+
+ # parse_config_directories currently only corrects directory names.
+ if parse:
+ logger.trace(f'Parsing config={config!r}')
+ with suppress(KeyError):
+ config['directories'] = parse_config_directories(config['directories'])
+ logger.trace(f'Config directories parsed: config={config!r}')
+
+ return config
+
+
+def save_config(save_path: Path, config: dict, overwrite: bool = True):
+ """Save config to local yaml file.
+
+ Args:
+ save_path (str): Path to save, can be relative or absolute. See Notes in
+ ``load_config``.
+ config (dict): Config to save.
+ overwrite (bool, optional): True if file should be updated, False
+ to generate a warning for existing config. Defaults to True
+ for updates.
+
+ Returns:
+ bool: If the save was successful.
+
+ Raises:
+ FileExistsError: If the local path already exists and ``overwrite=False``.
+ """
+ # Make sure it's a path.
+ save_path = Path(save_path)
+
+ # Make sure ends with '_local.yaml'.
+ if save_path.stem.endswith('_local') is False:
+ save_path = save_path.with_name(save_path.stem + '_local.yaml')
+
+ if save_path.exists() and overwrite is False:
+ raise FileExistsError(f"Path exists and overwrite=False: {save_path}")
+ else:
+ # Create directory if it does not exist.
+ save_path.parent.mkdir(parents=True, exist_ok=True)
+ logger.info(f'Saving config to {save_path}')
+ with save_path.open('w') as fn:
+ to_yaml(config, stream=fn)
+ logger.info(f'Config info saved to {save_path}')
+
+ return True
+
+
+def parse_config_directories(directories: Dict[str, str]):
+ """Parse the config dictionary for common objects.
+
+ Given a `base` entry that corresponds to the absolute path of a directory,
+ prepend the `base` to all other relative directory entries.
+
+ The `base` directory must exist or an exception is rasied.
+
+ If the `base` entry is not given the current working directory is used.
+
+ .. doctest::
+
+ >>> dirs_config = dict(base='/tmp', foo='bar', baz='bam', app='/app')
+ >>> parse_config_directories(dirs_config)
+ {'base': '/tmp', 'foo': '/tmp/bar', 'baz': '/tmp/bam', 'app': '/app'}
+
+ >>> # If base doesn't exist an exception is raised.
+ >>> dirs_config = dict(base='/panoptes', foo='bar', baz='bam', app='/app')
+ >>> parse_config_directories(dirs_config)
+ Traceback (most recent call last):
+ ...
+ error.NotFound: NotFound: Base directory does not exist: /panoptes
+
+ Args:
+ directories (dict): The dictionary of directory information. Usually comes
+ from the "directories" entry in the config.
+
+ Returns:
+ dict: The same directory but with relative directories resolved.
+
+ Raises:
+ error.NotFound: if the 'base' entry is given but does not exist.
+ """
+ resolved_dirs = directories.copy()
+
+ # Try to get the base directory first.
+ base_dir = Path(resolved_dirs.get('base', '.')).absolute()
+
+ # Warn if base directory does not exist.
+ if base_dir.is_dir() is False:
+ raise error.NotFound(f'Base directory does not exist: {base_dir}')
+
+ # Add back absolute path for base directory.
+ resolved_dirs['base'] = str(base_dir)
+ logger.trace(f'Using base_dir={base_dir!r} for setting config directories')
+
+ # Add the base directory to any relative dir.
+ for dir_name, dir_path in resolved_dirs.items():
+ if dir_path.startswith('/') is False and dir_name != 'base':
+ sub_dir = (base_dir / dir_path).absolute()
+
+ if sub_dir.exists() is False:
+ logger.warning(f'{sub_dir!r} does not exist.')
+
+ logger.trace(f'Setting {dir_name} to {sub_dir}')
+ resolved_dirs[dir_name] = str(sub_dir)
+
+ return resolved_dirs
+
+
+def _add_to_conf(config: dict, conf_fn: Path, parse: bool = False):
+ with suppress(IOError, TypeError):
+ with conf_fn.open('r') as fn:
+ config.update(from_yaml(fn.read(), parse=parse))
diff --git a/config/server.py b/config/server.py
new file mode 100644
index 00000000..a145c1f3
--- /dev/null
+++ b/config/server.py
@@ -0,0 +1,324 @@
+import logging
+import os
+from sys import platform
+from multiprocessing import Process
+
+from flask import Flask
+from flask import jsonify
+from flask import request
+from gevent.pywsgi import WSGIServer
+from scalpl import Cut
+
+from config.helpers import load_config
+from config.helpers import save_config
+
+# This seems to be needed. Should switch entire mechanism.
+if platform == "darwin" or platform == "win32":
+ import multiprocessing
+ multiprocessing.set_start_method('fork')
+
+# Turn off noisy logging for Flask wsgi server.
+logging.getLogger('werkzeug').setLevel(logging.WARNING)
+logging.getLogger('gevent').setLevel(logging.WARNING)
+# Setup logging
+logging.basicConfig(level=logging.DEBUG, format='%(asctime)s;%(levelname)s:%(message)s')
+logger = logging.getLogger(__name__)
+
+app = Flask(__name__)
+
+
+def config_server(config_file,
+ host=None,
+ port=None,
+ load_local=True,
+ save_local=False,
+ auto_start=True,
+ access_logs=None,
+ error_logs='logger',
+ ):
+ """Start the config server in a separate process.
+
+ A convenience function to start the config server.
+
+ Args:
+ config_file (str or None): The absolute path to the config file to load.
+ host (str, optional): The config server host. First checks for CONFIG_HOST
+ env var, defaults to 'localhost'.
+ port (str or int, optional): The config server port. First checks for CONFIG_HOST
+ env var, defaults to 6563.
+ load_local (bool, optional): If local config files should be used when loading, default True.
+ save_local (bool, optional): If setting new values should auto-save to local file, default False.
+ auto_start (bool, optional): If server process should be started automatically, default True.
+ access_logs ('default' or `logger` or `File`-like or None, optional): Controls access logs for
+ the gevent WSGIServer. The `default` string will cause access logs to go to stderr. The
+ string `logger` will use the logger. A File-like will write to file. The default
+ `None` will turn off all access logs.
+ error_logs ('default' or 'logger' or `File`-like or None, optional): Same as `access_logs` except we use
+ our `logger` as the default.
+
+ Returns:
+ multiprocessing.Process: The process running the config server.
+ """
+ logger.info(f'Starting config-server with config_file={config_file!r}')
+ config = load_config(config_files=config_file, load_local=load_local, parse=False)
+ logger.info(f'Config server Loaded {len(config)} top-level items')
+
+ # Add an entry to control running of the server.
+ config['config_server'] = dict(running=True)
+
+ logger.info(f'{config!r}')
+ cut_config = Cut(config)
+
+ with app.app_context():
+ app.config['config_file'] = config_file
+ app.config['save_local'] = save_local
+ app.config['load_local'] = load_local
+ app.config['POCS'] = config
+ app.config['POCS_cut'] = cut_config
+ logger.info(f'Config items saved to flask config-server')
+
+ # Set up access and error logs for server.
+ access_logs = logger if access_logs == 'logger' else access_logs
+ error_logs = logger if error_logs == 'logger' else error_logs
+
+ def start_server(host='localhost', port=6563):
+ try:
+ logger.info(f'Starting config server with {host}:{port}')
+ http_server = WSGIServer((host, int(port)), app, log=access_logs,
+ error_log=error_logs)
+ http_server.serve_forever()
+ except OSError:
+ logger.warning(
+ f'Problem starting config server, is another config server already running?')
+ return None
+ except Exception as e:
+ logger.warning(f'Problem starting config server: {e!r}')
+ return None
+
+ host = host or os.getenv('CONFIG_HOST', 'localhost')
+ port = port or os.getenv('CONFIG_PORT', 6563)
+ cmd_kwargs = dict(host=host, port=port)
+ logger.debug(f'Setting up config server process with cmd_kwargs={cmd_kwargs!r}')
+ server_process = Process(target=start_server,
+ daemon=True,
+ kwargs=cmd_kwargs)
+
+ if auto_start:
+ server_process.start()
+
+ return server_process
+
+
+@app.route('/heartbeat', methods=['GET', 'POST'])
+def heartbeat():
+ """A simple echo service to be used for a heartbeat.
+
+ Defaults to looking for the 'config_server.running' bool value, although a
+ different `key` can be specified in the POST.
+ """
+ params = dict()
+ if request.method == 'GET':
+ params = request.args
+ elif request.method == 'POST':
+ params = request.get_json()
+
+ key = params.get('key', 'config_server.running')
+ if key is None:
+ key = 'config_server.running'
+ is_running = app.config['POCS_cut'].get(key, False)
+
+ return jsonify(is_running)
+
+
+@app.route('/get-config', methods=['GET', 'POST'])
+def get_config_entry():
+ """Get config entries from server.
+
+ Endpoint that responds to GET and POST requests and returns
+ configuration item corresponding to provided key or entire
+ configuration. The key entries should be specified in dot-notation,
+ with the names corresponding to the entries stored in the configuration
+ file. See the `scalpl `_ documentation
+ for details on the dot-notation.
+
+ The endpoint should receive a JSON document with a single key named ``"key"``
+ and a value that corresponds to the desired key within the configuration.
+
+ For example, take the following configuration:
+
+ .. code:: javascript
+
+ {
+ 'location': {
+ 'elevation': 3400.0,
+ 'latitude': 19.55,
+ 'longitude': 155.12,
+ }
+ }
+
+ To get the corresponding value for the elevation, pass a JSON document similar to:
+
+ .. code:: javascript
+
+ '{"key": "location.elevation"}'
+
+ Returns:
+ str: The json string for the requested object if object is found in config.
+ Otherwise a json string with ``status`` and ``msg`` keys will be returned.
+ """
+ params = dict()
+ if request.method == 'GET':
+ params = request.args
+ elif request.method == 'POST':
+ params = request.get_json()
+
+ verbose = params.get('verbose', True)
+ log_level = 'DEBUG' if verbose else 'TRACE'
+
+ # If requesting specific key
+ logger.debug(f'Received params={params!r}')
+
+ if request.is_json:
+ try:
+ key = params['key']
+ logger.debug(f'Request contains key={key!r}')
+ except KeyError:
+ return jsonify({
+ 'success': False,
+ 'msg': "No valid key found. Need json request: {'key': }"
+ })
+
+ if key is None:
+ # Return all
+ logger.debug('No valid key given, returning entire config')
+ show_config = app.config['POCS']
+ else:
+ try:
+ logger.debug(f'Looking for key={key!r} in config')
+ show_config = app.config['POCS_cut'].get(key, None)
+ except Exception as e:
+ logger.error(f'Error while getting config item: {e!r}')
+ show_config = None
+ else:
+ # Return entire config
+ logger.debug('No valid key given, returning entire config')
+ show_config = app.config['POCS']
+
+ logger.debug(f'Returning show_config={show_config!r}')
+ logger.debug(f'Returning {show_config!r}')
+ return jsonify(show_config)
+
+
+@app.route('/set-config', methods=['GET', 'POST'])
+def set_config_entry():
+ """Sets an item in the config.
+
+ Endpoint that responds to GET and POST requests and sets a
+ configuration item corresponding to the provided key.
+
+ The key entries should be specified in dot-notation, with the names
+ corresponding to the entries stored in the configuration file. See
+ the `scalpl `_ documentation for details
+ on the dot-notation.
+
+ The endpoint should receive a JSON document with a single key named ``"key"``
+ and a value that corresponds to the desired key within the configuration.
+
+ For example, take the following configuration:
+
+ .. code:: javascript
+
+ {
+ 'location': {
+ 'elevation': 3400.0,
+ 'latitude': 19.55,
+ 'longitude': 155.12,
+ }
+ }
+
+ To set the corresponding value for the elevation, pass a JSON document similar to:
+
+ .. code:: javascript
+
+ '{"location.elevation": "1000 m"}'
+
+
+ Returns:
+ str: If method is successful, returned json string will be a copy of the set values.
+ On failure, a json string with ``status`` and ``msg`` keys will be returned.
+ """
+ params = dict()
+ if request.method == 'GET':
+ params = request.args
+ elif request.method == 'POST':
+ params = request.get_json()
+
+ if params is None:
+ return jsonify({
+ 'success': False,
+ 'msg': "Invalid. Need json request: {'key': , 'value': }"
+ })
+
+ try:
+ app.config['POCS_cut'].update(params)
+ except KeyError:
+ for k, v in params.items():
+ app.config['POCS_cut'].setdefault(k, v)
+
+ # Config has been modified so save to file.
+ save_local = app.config['save_local']
+ logger.info(f'Setting config save_local={save_local!r}')
+ if save_local and app.config['config_file'] is not None:
+ save_config(app.config['config_file'], app.config['POCS_cut'].copy())
+
+ return jsonify(params)
+
+
+@app.route('/reset-config', methods=['POST'])
+def reset_config():
+ """Reset the configuration.
+
+ An endpoint that accepts a POST method. The json request object
+ must contain the key ``reset`` (with any value).
+
+ The method will reset the configuration to the original configuration files that were
+ used, skipping the local (and saved file).
+
+ .. note::
+
+ If the server was originally started with a local version of the file, those will
+ be skipped upon reload. This is not ideal but hopefully this method is not used too
+ much.
+
+ Returns:
+ str: A json string object containing the keys ``success`` and ``msg`` that indicate
+ success or failure.
+ """
+ params = dict()
+ if request.method == 'GET':
+ params = request.args
+ elif request.method == 'POST':
+ params = request.get_json()
+
+ logger.warning(f'Resetting config server')
+
+ if params['reset']:
+ # Reload the config
+ config = load_config(config_files=app.config['config_file'],
+ load_local=app.config['load_local'],
+ parse=params.get('parse', False)
+ )
+ # Add an entry to control running of the server.
+ config['config_server'] = dict(running=True)
+ app.config['POCS'] = config
+ app.config['POCS_cut'] = Cut(config)
+ else:
+ return jsonify({
+ 'success': False,
+ 'msg': "Invalid. Need json request: {'reset': True}"
+ })
+
+ return jsonify({
+ 'success': True,
+ 'msg': f'Configuration reset'
+ })
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 59432b2c..7e982aca 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -32,4 +32,13 @@ Sphinx==1.8.1
sqlmodel==0.0.22
transitions==0.7.1
tzwhere==3.0.3
-watchdog==4.0.1
\ No newline at end of file
+watchdog==4.0.1
+
+
+gevent==25.4.2
+Flask==3.1.0
+gevent==25.4.2
+Flask==3.1.0
+scalpl==0.4.2
+serializers==0.2.4
+ruamel.yaml==0.18.10
\ No newline at end of file
diff --git a/utils/serializers.py b/utils/serializers.py
index 79c27335..2f3fb7c0 100644
--- a/utils/serializers.py
+++ b/utils/serializers.py
@@ -1,66 +1,389 @@
-# Generic stuff
-from bson import json_util
+import json
+from collections import OrderedDict
+from contextlib import suppress
+from copy import deepcopy
+import numpy as np
+from astropy import units as u
+from astropy.time import Time
+from dateutil.parser import isoparse as date_parse
+from utils import error
+from ruamel.yaml import YAML
+from ruamel.yaml.compat import StringIO
-def dumps(obj):
- """Dump an object to JSON.
+
+class StringYAML(YAML):
+ def dump(self, data, stream=None, **kwargs):
+ """YAML class that can dump to a string.
+
+ By default the YAML parser doesn't serialize directly to a string. This
+ class is a small wrapper to output StreamIO as a string if no stream is
+ provided.
+
+ See https://yaml.readthedocs.io/en/latest/example.html#output-of-dump-as-a-string.
+
+ Note:
+
+ This class should not be used directly but instead is instantiated as
+ part of the yaml convenience methods below.
+
+ Args:
+ data (`object`): An object, usually dict-like.
+ stream (`None` | stream, optional): A stream object to write the YAML.
+ If default `None`, return value as string.
+ **kwargs: Keywords passed to the `dump` function.
+
+ Returns:
+ `str`: The serialized object string.
+ """
+ inefficient = False
+ if stream is None:
+ inefficient = True
+ stream = StringIO()
+ yaml = YAML()
+ yaml.dump(data, stream, **kwargs)
+ if inefficient:
+ return stream.getvalue()
+
+
+def to_json(obj, filename=None, append=True, **kwargs):
+ """Convert a Python object to a JSON string.
+
+ Will handle `datetime` objects as well as `astropy.unit.Quantity` objects.
+ Astropy quantities will be converted to a dict: `{"value": val, "unit": unit}`.
+
+ Examples:
+
+ .. doctest::
+
+ >>> from utils.serializers import to_json
+ >>> from astropy import units as u
+ >>> config = { "name": "Mauna Loa", "elevation": 3397 * u.meter }
+ >>> to_json(config)
+ '{"name": "Mauna Loa", "elevation": "3397.0 m"}'
+
+ >>> to_json({"numpy_array": np.arange(10)})
+ '{"numpy_array": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}'
+
+ >>> from utils.time import current_time
+ >>> to_json({"current_time": current_time()}) # doctest: +SKIP
+ '{"current_time": "2019-04-08 22:19:28.402198"}'
Args:
- obj (dict): An object to serialize.
+ obj (`object`): The object to be converted to JSON, usually a dict.
+ filename (`str`, optional): Path to file for saving.
+ append (`bool`, optional): Append to `filename`, default True. Setting
+ False will clobber the file.
+ **kwargs: Keyword arguments passed to `json.dumps`.
Returns:
- str: Serialized representation of object.
+ `str`: The JSON string representation of the object.
"""
- return json_util.dumps(obj)
+ try:
+ json_str = json.dumps(obj, default=serialize_object, **kwargs)
+ except Exception as e:
+ raise error.InvalidSerialization(e)
+
+ if filename is not None:
+ mode = 'w'
+ if append:
+ mode = 'a'
+ with open(filename, mode) as fn:
+ fn.write(json_str + '\n')
+
+ return json_str
+
+
+def from_json(msg):
+ """Convert a JSON string into a Python object.
+
+ Astropy quanitites will be converted from a ``{"value": val, "unit": unit}``
+ format. Additionally, the following units will be converted if the value ends
+ with the exact string:
+
+ * deg
+ * m
+ * s
+
+ Time-like values are *not* parsed, however see example below.
+
+ Examples:
+
+ .. doctest::
+
+ >>> from utils.serializers import from_json
+ >>> config_str = '{"name":"Mauna Loa","elevation":{"value":3397.0,"unit":"m"}}'
+ >>> from_json(config_str)
+ {'name': 'Mauna Loa', 'elevation': }
+ # Invalid values will be returned as is.
+ >>> from_json('{"horizon":{"value":42.0,"unit":"degr"}}')
+ {'horizon': {'value': 42.0, 'unit': 'degr'}}
-def loads(msg):
- """Load an object from JSON.
+ # The following will convert if final string:
+ >>> from_json('{"horizon": "42.0 deg"}')
+ {'horizon': }
+
+ >>> from_json('{"elevation": "1000 m"}')
+ {'elevation': }
+
+ >>> from_json('{"readout_time": "10 s"}')
+ {'readout_time': }
+
+ # Be careful with short unit names in extended format!
+ >>> horizon = from_json('{"horizon":{"value":42.0,"unit":"d"}}')
+ >>> horizon['horizon']
+
+ >>> horizon['horizon'].decompose()
+
+
+ >>> from utils.time import current_time
+ >>> time_str = to_json({"current_time": current_time().datetime})
+ >>> from_json(time_str)['current_time'] # doctest: +SKIP
+ 2019-04-08T06:43:28.232406
+
+ >>> from astropy.time import Time
+ >>> Time(from_json(time_str)['current_time']) # doctest: +SKIP
+