diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2d77c721f..1a32f0566 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -208,7 +208,7 @@ v0.3.0 * ``zip`` and ``state`` only apply to ``USA`` institutions * added group item in people schema -* ``KeyError`` for ``ChainDB`` now prints the offending key +* ``KeyError`` for ``ChainDocument`` now prints the offending key None * preslist now includes end-dates when meeting is longer than one day diff --git a/docs/tutorials/broker.rst b/docs/tutorials/broker.rst index 0b7b1edfb..7b94f4be4 100644 --- a/docs/tutorials/broker.rst +++ b/docs/tutorials/broker.rst @@ -46,7 +46,7 @@ directory. Do not paste them into the _build folder, but rather the base directo plt.plot(range(10), range(10)) plt.savefig('hello_world.png') - doc = db['projects']['regro'] + doc = db['test_db']['projects']['regro'] db.add_file(doc, 'hw_file', 'hello_world.png') This will: @@ -68,7 +68,7 @@ Importantly, this will only retrieve the path to the file. from regolith.broker import Broker db = Broker.from_rc() - doc = db['projects']['regro'] + doc = db['test_db']['projects']['regro'] path = db.get_file_path(doc, 'hw_file') This can be used inside tex documents via the ``FigureBuilder`` class/CLI. In order to do so, diff --git a/regolith/broker.py b/regolith/broker.py index 9c89675a0..6da9bf44d 100644 --- a/regolith/broker.py +++ b/regolith/broker.py @@ -4,6 +4,7 @@ from regolith.database import dump_database, open_dbs from regolith.runcontrol import DEFAULT_RC, load_rcfile, filter_databases from regolith.storage import store_client, push +from regolith.mongoclient import load_mongo_col def load_db(rc_file="regolithrc.json"): @@ -23,7 +24,7 @@ class Broker: >>> # Load the db >>> db = Broker.from_rc() >>> # Get a docment from the broker - >>> ergs =db['group']['ergs'] + >>> ergs =db['test_db']['group']['ergs'] >>> # Store a file >>> db.add_file(ergs, 'myfile', '/path/to/file/hello.txt') >>> # Get a file from the store @@ -36,6 +37,10 @@ def __init__(self, rc=DEFAULT_RC): with store_client(rc) as sclient: self.store = sclient rc.client = open_dbs(rc) + for name, dbs in rc.client.dbs.items(): + for coll in dbs: + if not isinstance(dbs[coll], dict): + dbs[coll] = load_mongo_col(dbs[coll]) self._dbs = rc.client.dbs self.md = rc.client.chained_db self.db_client = rc.client @@ -88,4 +93,4 @@ def get_file_path(self, document, name): return None def __getitem__(self, item): - return self.md[item] + return self._dbs[item] diff --git a/regolith/builders/activitylogbuilder.py b/regolith/builders/activitylogbuilder.py index 1f2d808de..f746a43eb 100644 --- a/regolith/builders/activitylogbuilder.py +++ b/regolith/builders/activitylogbuilder.py @@ -6,7 +6,6 @@ from regolith.builders.basebuilder import LatexBuilderBase from regolith.builders.cpbuilder import is_pending -from regolith.fsclient import _id_key from regolith.dates import month_to_int, is_current, get_dates from regolith.sorters import position_key, doc_date_key from regolith.stylers import sentencecase, month_fullnames @@ -25,7 +24,7 @@ awards, filter_patents, filter_licenses, - get_id_from_name, merge_collections_all, filter_committees) + get_id_from_name, merge_collections_all, filter_committees, _id_key) class ActivitylogBuilder(LatexBuilderBase): diff --git a/regolith/builders/appraisalbuilder.py b/regolith/builders/appraisalbuilder.py index 90934fb3a..e5b417631 100644 --- a/regolith/builders/appraisalbuilder.py +++ b/regolith/builders/appraisalbuilder.py @@ -4,7 +4,6 @@ from regolith.builders.basebuilder import LatexBuilderBase from regolith.builders.cpbuilder import is_pending, CPBuilder -from regolith.fsclient import _id_key from regolith.dates import month_to_int, is_current, get_dates from regolith.sorters import position_key, doc_date_key from regolith.stylers import sentencecase, month_fullnames @@ -24,7 +23,10 @@ filter_patents, filter_licenses, merge_collections_superior, - get_id_from_name, merge_collections_all) + get_id_from_name, + merge_collections_all, + _id_key +) class AppraisalBuilder(LatexBuilderBase): diff --git a/regolith/builders/cpbuilder.py b/regolith/builders/cpbuilder.py index 950a1accc..8c48c2f09 100644 --- a/regolith/builders/cpbuilder.py +++ b/regolith/builders/cpbuilder.py @@ -5,13 +5,13 @@ from regolith.builders.basebuilder import LatexBuilderBase from regolith.dates import is_current, get_dates -from regolith.fsclient import _id_key from regolith.sorters import position_key from regolith.tools import ( all_docs_from_collection, filter_grants, fuzzy_retrieval, merge_collections_all, + _id_key ) diff --git a/regolith/builders/cvbuilder.py b/regolith/builders/cvbuilder.py index e08ed776d..667015ee6 100644 --- a/regolith/builders/cvbuilder.py +++ b/regolith/builders/cvbuilder.py @@ -3,7 +3,6 @@ from datetime import date from regolith.builders.basebuilder import LatexBuilderBase -from regolith.fsclient import _id_key from regolith.sorters import ene_date_key, position_key from regolith.stylers import sentencecase, month_fullnames from regolith.tools import ( @@ -17,6 +16,7 @@ dereference_institution, merge_collections_superior, filter_presentations, remove_duplicate_docs, + _id_key ) diff --git a/regolith/builders/htmlbuilder.py b/regolith/builders/htmlbuilder.py index 1a3e922af..fe4b058fd 100644 --- a/regolith/builders/htmlbuilder.py +++ b/regolith/builders/htmlbuilder.py @@ -4,7 +4,6 @@ from regolith.builders.basebuilder import BuilderBase from regolith.dates import get_dates -from regolith.fsclient import _id_key from regolith.sorters import ene_date_key, position_key from regolith.tools import ( all_docs_from_collection, @@ -13,6 +12,7 @@ make_bibtex_file, document_by_value, dereference_institution, + _id_key ) diff --git a/regolith/builders/internalhtmlbuilder.py b/regolith/builders/internalhtmlbuilder.py index 89f7b6bf6..9fa09843c 100644 --- a/regolith/builders/internalhtmlbuilder.py +++ b/regolith/builders/internalhtmlbuilder.py @@ -5,7 +5,6 @@ from regolith.builders.basebuilder import BuilderBase from regolith.dates import get_dates -from regolith.fsclient import _id_key from regolith.sorters import position_key, ene_date_key from regolith.tools import ( all_docs_from_collection, @@ -15,7 +14,8 @@ make_bibtex_file, document_by_value, dereference_institution, - fuzzy_retrieval + fuzzy_retrieval, + _id_key ) diff --git a/regolith/builders/manuscriptreviewbuilder.py b/regolith/builders/manuscriptreviewbuilder.py index 65e67df70..6f5500eb2 100644 --- a/regolith/builders/manuscriptreviewbuilder.py +++ b/regolith/builders/manuscriptreviewbuilder.py @@ -1,9 +1,9 @@ """Builder for Current and Pending Reports.""" from regolith.builders.basebuilder import LatexBuilderBase -from regolith.fsclient import _id_key from regolith.tools import ( all_docs_from_collection, + _id_key ) diff --git a/regolith/builders/postdocadbuilder.py b/regolith/builders/postdocadbuilder.py index 048433614..8001da247 100644 --- a/regolith/builders/postdocadbuilder.py +++ b/regolith/builders/postdocadbuilder.py @@ -1,9 +1,9 @@ """Builder for Current and Pending Reports.""" from regolith.builders.basebuilder import LatexBuilderBase -from regolith.fsclient import _id_key from regolith.tools import ( all_docs_from_collection, + _id_key ) diff --git a/regolith/builders/preslistbuilder.py b/regolith/builders/preslistbuilder.py index cb4fc9ef2..6a93cddf1 100644 --- a/regolith/builders/preslistbuilder.py +++ b/regolith/builders/preslistbuilder.py @@ -21,14 +21,16 @@ from copy import deepcopy from regolith.builders.basebuilder import LatexBuilderBase -from regolith.fsclient import _id_key from regolith.sorters import position_key from regolith.tools import ( all_docs_from_collection, fuzzy_retrieval, get_person_contact, number_suffix, - group_member_ids, latex_safe, filter_presentations + group_member_ids, + latex_safe, + filter_presentations, + _id_key ) from regolith.stylers import sentencecase, month_fullnames from regolith.dates import get_dates diff --git a/regolith/builders/proposalreviewbuilder.py b/regolith/builders/proposalreviewbuilder.py index a7a572489..c11a8b587 100644 --- a/regolith/builders/proposalreviewbuilder.py +++ b/regolith/builders/proposalreviewbuilder.py @@ -2,11 +2,12 @@ from nameparser import HumanName from regolith.builders.basebuilder import LatexBuilderBase -from regolith.fsclient import _id_key from regolith.tools import ( all_docs_from_collection, filter_grants, - fuzzy_retrieval, dereference_institution, + fuzzy_retrieval, + dereference_institution, + _id_key ) diff --git a/regolith/builders/readinglistsbuilder.py b/regolith/builders/readinglistsbuilder.py index 573972ce9..7fb4417d4 100644 --- a/regolith/builders/readinglistsbuilder.py +++ b/regolith/builders/readinglistsbuilder.py @@ -3,11 +3,11 @@ from habanero import Crossref from regolith.builders.basebuilder import LatexBuilderBase -from regolith.fsclient import _id_key from regolith.sorters import position_key from regolith.tools import ( all_docs_from_collection, get_formatted_crossref_reference, + _id_key ) class ReadingListsBuilder(LatexBuilderBase): diff --git a/regolith/chained_db.py b/regolith/chained_db.py index b694f0260..5e0b36143 100644 --- a/regolith/chained_db.py +++ b/regolith/chained_db.py @@ -8,6 +8,11 @@ from collections import ChainMap from collections.abc import MutableMapping +from copy import deepcopy + +from pymongo.collection import Collection as MongoCollection + +from regolith.mongoclient import load_mongo_col class ChainDBSingleton(object): @@ -24,8 +29,91 @@ def __new__(cls): Singleton = ChainDBSingleton() -class ChainDB(ChainMap): - """ A ChainMap who's ``_getitem__`` returns either a ChainDB or +class ChainCollection: + """ + The chained database has been used as a reference up until the transition to remote mongo, rather than an edit-able + object. When using only remote mongo databases, the chained database will become a fully functional chained mongo + collection rather than a reference of what all of the databases look like mashed together. Checking to see if the + object's fs_map dictionary is empty is a good indicator of whether or not mongo methods can/should be used directly. + + e.g. + if rc.client.chained_db[collection_name].fs_map == {}: + rc.client.chained_db[collection_name].find_one_and_update({"keyName": "Value"},{"$set": {UpdateDict}}) + """ + + def __init__(self, *maps): + ''' + Initialize a ChainCollection by setting *maps* to the given mappings. + ''' + + self.mongo_maps = [] + self.fs_map = {} + + map_list = list(maps) + + if map_list and all([isinstance(map, MongoCollection) for map in map_list]): + self.mongo_maps = map_list + elif len(map_list) == 1 and isinstance(map_list[0], dict): + # There is only ever one collection for the filesystem, + # as it is chained at the document level, not collection + self.fs_map = deepcopy(map_list[0]) + + def __iter__(self): + # load all docs from each mongo map, create list of dicts, chainmap them together, get iter of the chainmap + mongo_chain = ChainMap(*reversed([load_mongo_col(collection) for collection in self.mongo_maps])) + fs_chain = self.fs_map + return iter(ChainMap(*[mongo_chain, fs_chain])) + + def __getitem__(self, doc_id): + chained_mongo_docs = ChainMap(*reversed([collection.find_one({"_id": doc_id}) for collection in self.mongo_maps])) + chained_fs_docs = self.fs_map.get(doc_id, {}) + return ChainMap(*[chained_mongo_docs, chained_fs_docs]) + + def __setitem__(self, doc_id, document): + if isinstance(document, ChainDocument): + self.fs_map[doc_id] = deepcopy(document) + elif self.mongo_maps: + # reached if mongo maps is not empty and the document is not chained + for db_coll in self.mongo_maps: + db_coll.find_one_and_update({"_id": doc_id}, {"$set": document}) + + def __getattr__(self, method): + if hasattr(MongoCollection, method): + results = [] + for db_coll in self.mongo_maps: + results.append(getattr(db_coll, method)) + + # This is a closure that forces the evaluation of the mongo method on every collection w/ the same name. + def multi_call(args): + mongo_results = [] + for result in results: + mongo_results.extend(result(args)) + return mongo_results + + return multi_call + else: + raise AttributeError + + def keys(self): + if self.fs_map: + key_list = self.mongo_maps + [self.fs_map] + else: + key_list = self.mongo_maps + return ChainMap(*reversed(key_list)).keys() + + def values(self): + if self.fs_map: + full_coll = [load_mongo_col(collection) for collection in self.mongo_maps] + [self.fs_map] + else: + full_coll = [load_mongo_col(collection) for collection in self.mongo_maps] + return ChainMap(*reversed(full_coll)).values() + + def items(self): + return zip(self.keys(), self.values()) + + +class ChainDocument(ChainMap): + """ A ChainMap who's ``_getitem__`` returns either a ChainDocument or the result""" def __getitem__(self, key): @@ -34,11 +122,11 @@ def __getitem__(self, key): # Try to get all the data from all the mappings for mapping in self.maps: results.append(mapping.get(key, Singleton)) - # if all the results are mapping create a ChainDB + # if all the results are mapping create a ChainDocument if all([isinstance(result, MutableMapping) for result in results]): for result in results: if res is None: - res = ChainDB(result) + res = ChainDocument(result) else: res.maps.append(result) elif all([isinstance(result, list) for result in results]): @@ -62,16 +150,15 @@ def __setitem__(self, key, value): if key not in self: super().__setitem__(key, value) else: - res = None - results = [] # Try to get all the data from all the mappings for mapping in reversed(self.maps): if key in mapping: mapping[key] = value + def _convert_to_dict(cm): - if isinstance(cm, (ChainMap, ChainDB)): + if isinstance(cm, (ChainMap, ChainDocument)): r = {} for k, v in cm.items(): r[k] = _convert_to_dict(v) diff --git a/regolith/client_manager.py b/regolith/client_manager.py index f7c8ba30c..3413639e0 100644 --- a/regolith/client_manager.py +++ b/regolith/client_manager.py @@ -2,7 +2,7 @@ from collections import defaultdict from regolith.fsclient import FileSystemClient -from regolith.mongoclient import MongoClient +from regolith.mongoclient import MongoClient, load_mongo_col CLIENTS = { @@ -99,9 +99,14 @@ def collection_names(self, dbname, include_system_collections=True): def all_documents(self, collname, copy=True): """Returns an iteratable over all documents in a collection.""" - if copy: - return deepcopy(self.chained_db.get(collname, {})).values() - return self.chained_db.get(collname, {}).values() + if isinstance(self.chained_db.get(collname, {}), dict): + if copy: + return deepcopy(self.chained_db.get(collname, {})).values() + return self.chained_db.get(collname, {}).values() + else: + # assume we've got a mongo collection + mongo_col = self.chained_db.get(collname, {}).values() + return mongo_col def insert_one(self, dbname, collname, doc): """Inserts one document to a database/collection.""" diff --git a/regolith/database.xsh b/regolith/database.xsh index 0ade27048..6a52f9480 100644 --- a/regolith/database.xsh +++ b/regolith/database.xsh @@ -2,6 +2,7 @@ import os from contextlib import contextmanager from warnings import warn +from copy import deepcopy from xonsh.lib import subprocess from xonsh.lib.os import indir @@ -11,7 +12,7 @@ try: except: hglib = None -from regolith.chained_db import ChainDB +from regolith.chained_db import ChainDocument, ChainCollection from regolith.tools import dbdirname from regolith.client_manager import ClientManager @@ -145,14 +146,14 @@ def dump_database(db, client, rc): raise ValueError('Do not know how to dump this kind of database') -def open_dbs(rc, dbs=None): +def open_dbs(rc, colls=None): """Open the databases Parameters ---------- rc : RunControl instance The rc which has links to the dbs - dbs: set or None, optional + colls: set or None, optional The databases to load. If None load all, defaults to None Returns @@ -160,33 +161,39 @@ def open_dbs(rc, dbs=None): client : {FileSystemClient, MongoClient} The database client """ - if dbs is None: - dbs = [] + if colls is None: + colls = [] client = ClientManager(rc.databases, rc) client.open() chained_db = {} for db in rc.databases: - # if we only want to access some dbs and this db is not in that some - db['whitelist'] = dbs + # if we only want to access some colls and this db is not in that some + db['whitelist'] = colls if 'blacklist' not in db: db['blacklist'] = ['.travis.yml', '.travis.yaml'] load_database(db, client, rc) for base, coll in client.dbs[db['name']].items(): - if base not in chained_db: - chained_db[base] = {} - for k, v in coll.items(): - if k in chained_db[base]: - chained_db[base][k].maps.append(v) + if isinstance(coll, dict): + if base not in chained_db: + chained_db[base] = ChainCollection() + for k, v in coll.items(): + if k in chained_db[base].fs_map: + chained_db[base].fs_map[k].maps.append(v) + else: + chained_db[base].fs_map[k] = deepcopy(ChainDocument(v)) + else: + if base not in chained_db: + chained_db[base] = ChainCollection(coll) else: - chained_db[base][k] = ChainDB(v) + chained_db[base].mongo_maps.append(coll) client.chained_db = chained_db return client @contextmanager -def connect(rc, dbs=None): +def connect(rc, colls=None): """Context manager for ensuring that database is properly setup and torn down""" - client = open_dbs(rc, dbs=dbs) + client = open_dbs(rc, colls=colls) yield client for db in rc.databases: dump_database(db, client, rc) diff --git a/regolith/fsclient.py b/regolith/fsclient.py index c3e63082a..f7f224d7c 100644 --- a/regolith/fsclient.py +++ b/regolith/fsclient.py @@ -11,7 +11,7 @@ from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap, CommentedSeq -from regolith.tools import dbpathname +from regolith.tools import dbpathname, _id_key import signal import logging @@ -53,10 +53,6 @@ def _rec_re_type(i): return base -def _id_key(doc): - return doc["_id"] - - def load_json(filename): """Loads a JSON file and returns a dict of its documents.""" docs = {} @@ -185,6 +181,9 @@ def load_yaml(self, db, dbpath): def load_database(self, db): """Loads a database.""" dbpath = dbpathname(db, self.rc) + if db["name"] in self.dbs: + print("WARNING: Having two databases with the same name will result in certain changes only happening to" + "the latter.") self.load_json(db, dbpath) self.load_yaml(db, dbpath) diff --git a/regolith/helpers/a_expensehelper.py b/regolith/helpers/a_expensehelper.py index 627f64ea1..ac91951e5 100644 --- a/regolith/helpers/a_expensehelper.py +++ b/regolith/helpers/a_expensehelper.py @@ -5,11 +5,11 @@ import dateutil.parser as date_parser from regolith.helpers.basehelper import DbHelperBase -from regolith.fsclient import _id_key from regolith.schemas import EXPENSES_STATI, EXPENSES_TYPES from regolith.tools import ( all_docs_from_collection, get_pi_id, + _id_key ) from gooey import GooeyParser diff --git a/regolith/helpers/a_grppub_readlisthelper.py b/regolith/helpers/a_grppub_readlisthelper.py index cb51722de..598e5b81c 100644 --- a/regolith/helpers/a_grppub_readlisthelper.py +++ b/regolith/helpers/a_grppub_readlisthelper.py @@ -3,9 +3,9 @@ import dateutil.parser as date_parser from regolith.helpers.basehelper import DbHelperBase -from regolith.fsclient import _id_key from regolith.tools import ( all_docs_from_collection, + _id_key ) from gooey import GooeyParser diff --git a/regolith/helpers/a_manurevhelper.py b/regolith/helpers/a_manurevhelper.py index 0575d01fa..8e188f99a 100644 --- a/regolith/helpers/a_manurevhelper.py +++ b/regolith/helpers/a_manurevhelper.py @@ -6,9 +6,9 @@ from regolith.helpers.basehelper import DbHelperBase from regolith.dates import month_to_str_int -from regolith.fsclient import _id_key from regolith.tools import ( all_docs_from_collection, + _id_key ) from gooey import GooeyParser diff --git a/regolith/helpers/a_presentationhelper.py b/regolith/helpers/a_presentationhelper.py index 9f05d971f..16b0a7a0e 100644 --- a/regolith/helpers/a_presentationhelper.py +++ b/regolith/helpers/a_presentationhelper.py @@ -4,11 +4,11 @@ from regolith.helpers.a_expensehelper import expense_constructor from regolith.helpers.basehelper import DbHelperBase -from regolith.fsclient import _id_key from regolith.schemas import PRESENTATION_TYPES, PRESENTATION_STATI from regolith.tools import ( all_docs_from_collection, get_pi_id, + _id_key ) from gooey import GooeyParser diff --git a/regolith/helpers/a_projectumhelper.py b/regolith/helpers/a_projectumhelper.py index d5bc4f757..9bee05118 100644 --- a/regolith/helpers/a_projectumhelper.py +++ b/regolith/helpers/a_projectumhelper.py @@ -8,10 +8,10 @@ from dateutil.relativedelta import relativedelta from regolith.helpers.basehelper import DbHelperBase -from regolith.fsclient import _id_key from regolith.tools import ( all_docs_from_collection, get_pi_id, + _id_key ) from gooey import GooeyParser diff --git a/regolith/helpers/a_proposalhelper.py b/regolith/helpers/a_proposalhelper.py index a69fa1eab..304216f38 100644 --- a/regolith/helpers/a_proposalhelper.py +++ b/regolith/helpers/a_proposalhelper.py @@ -6,10 +6,10 @@ from math import floor from regolith.helpers.basehelper import DbHelperBase -from regolith.fsclient import _id_key from regolith.tools import ( all_docs_from_collection, get_pi_id, + _id_key ) from gooey import GooeyParser diff --git a/regolith/helpers/a_proprevhelper.py b/regolith/helpers/a_proprevhelper.py index 0323d0d80..f0cc724aa 100644 --- a/regolith/helpers/a_proprevhelper.py +++ b/regolith/helpers/a_proprevhelper.py @@ -6,9 +6,9 @@ from regolith.helpers.basehelper import DbHelperBase from regolith.dates import month_to_str_int -from regolith.fsclient import _id_key from regolith.tools import ( all_docs_from_collection, + _id_key ) from gooey import GooeyParser @@ -59,10 +59,14 @@ def construct_global_ctx(self): if not rc.database: rc.database = rc.databases[0]["name"] rc.coll = "proposalReviews" - gtx["proposalReviews"] = sorted( - all_docs_from_collection(rc.client, "proposalReviews"), key=_id_key - ) - gtx["all_docs_from_collection"] = all_docs_from_collection + if rc.client.chained_db[rc.coll].fs_map != {}: + gtx[rc.coll] = sorted( + all_docs_from_collection(rc.client, rc.coll), key=_id_key + ) + gtx["all_docs_from_collection"] = all_docs_from_collection + else: + # assume it is a mongo collection + gtx[rc.coll] = rc.client.chained_db[rc.coll] gtx["float"] = float gtx["str"] = str gtx["zip"] = zip @@ -78,7 +82,11 @@ def db_updater(self): name.first.casefold().strip(".")) coll = self.gtx[rc.coll] - pdocl = list(filter(lambda doc: doc["_id"] == key, coll)) + if isinstance(coll, list): + pdocl = list(filter(lambda doc: doc["_id"] == key, coll)) + else: + # assume mongo collection + pdocl = coll.find({"_id": key}) if len(pdocl) > 0: sys.exit("This entry appears to already exist in the collection") else: diff --git a/regolith/helpers/a_todohelper.py b/regolith/helpers/a_todohelper.py index d86b51909..5a0085ed8 100644 --- a/regolith/helpers/a_todohelper.py +++ b/regolith/helpers/a_todohelper.py @@ -6,10 +6,10 @@ from dateutil.relativedelta import relativedelta from regolith.helpers.basehelper import DbHelperBase -from regolith.fsclient import _id_key from regolith.tools import ( all_docs_from_collection, get_pi_id, + _id_key ) from gooey import GooeyParser diff --git a/regolith/helpers/f_todohelper.py b/regolith/helpers/f_todohelper.py index b300091f0..e3310bb08 100644 --- a/regolith/helpers/f_todohelper.py +++ b/regolith/helpers/f_todohelper.py @@ -6,13 +6,13 @@ import math from regolith.helpers.basehelper import DbHelperBase -from regolith.fsclient import _id_key from regolith.tools import ( all_docs_from_collection, get_pi_id, document_by_value, print_task, - key_value_pair_filter + key_value_pair_filter, + _id_key ) from gooey import GooeyParser diff --git a/regolith/helpers/hellohelper.py b/regolith/helpers/hellohelper.py index 6cb871aa8..67b016f3e 100644 --- a/regolith/helpers/hellohelper.py +++ b/regolith/helpers/hellohelper.py @@ -1,9 +1,9 @@ """Builder for Current and Pending Reports.""" from regolith.helpers.basehelper import SoutHelperBase -from regolith.fsclient import _id_key from regolith.tools import ( all_docs_from_collection, + _id_key ) diff --git a/regolith/helpers/l_abstracthelper.py b/regolith/helpers/l_abstracthelper.py index 7d87a9495..cb223e686 100644 --- a/regolith/helpers/l_abstracthelper.py +++ b/regolith/helpers/l_abstracthelper.py @@ -1,10 +1,10 @@ from regolith.dates import get_dates from regolith.helpers.basehelper import SoutHelperBase -from regolith.fsclient import _id_key from regolith.tools import ( all_docs_from_collection, get_pi_id, - get_person_contact + get_person_contact, + _id_key ) from gooey import GooeyParser diff --git a/regolith/helpers/l_contactshelper.py b/regolith/helpers/l_contactshelper.py index 8a2d40601..b05379c17 100644 --- a/regolith/helpers/l_contactshelper.py +++ b/regolith/helpers/l_contactshelper.py @@ -8,12 +8,12 @@ get_dates ) from regolith.helpers.basehelper import SoutHelperBase -from regolith.fsclient import _id_key from regolith.tools import ( all_docs_from_collection, get_pi_id, fuzzy_retrieval, search_collection, + _id_key ) from gooey import GooeyParser diff --git a/regolith/helpers/l_generalhelper.py b/regolith/helpers/l_generalhelper.py index c28c9d2e0..90a527ab6 100644 --- a/regolith/helpers/l_generalhelper.py +++ b/regolith/helpers/l_generalhelper.py @@ -2,12 +2,12 @@ """ from regolith.helpers.basehelper import SoutHelperBase -from regolith.fsclient import _id_key from regolith.tools import ( all_docs_from_collection, get_pi_id, search_collection, collection_str, + _id_key ) HELPER_TARGET = "lister" diff --git a/regolith/helpers/l_grantshelper.py b/regolith/helpers/l_grantshelper.py index 6a5e7ff90..b06acabff 100644 --- a/regolith/helpers/l_grantshelper.py +++ b/regolith/helpers/l_grantshelper.py @@ -8,14 +8,14 @@ from regolith.dates import get_dates, is_current from regolith.helpers.basehelper import SoutHelperBase -from regolith.fsclient import _id_key from regolith.tools import ( all_docs_from_collection, get_pi_id, search_collection, key_value_pair_filter, collection_str, - merge_collections_superior + merge_collections_superior, + _id_key ) from gooey import GooeyParser diff --git a/regolith/helpers/l_membershelper.py b/regolith/helpers/l_membershelper.py index 2f3b3d0c9..30b0f2a3b 100644 --- a/regolith/helpers/l_membershelper.py +++ b/regolith/helpers/l_membershelper.py @@ -4,13 +4,13 @@ from regolith.dates import is_current from regolith.helpers.basehelper import SoutHelperBase -from regolith.fsclient import _id_key from regolith.sorters import position_key from regolith.tools import ( all_docs_from_collection, key_value_pair_filter, collection_str, get_pi_id, fuzzy_retrieval, + _id_key ) from regolith.dates import get_dates diff --git a/regolith/helpers/l_milestoneshelper.py b/regolith/helpers/l_milestoneshelper.py index 2c3ee9ed6..ccf45a045 100644 --- a/regolith/helpers/l_milestoneshelper.py +++ b/regolith/helpers/l_milestoneshelper.py @@ -6,12 +6,12 @@ from regolith.dates import get_due_date from regolith.helpers.basehelper import SoutHelperBase -from regolith.fsclient import _id_key from regolith.tools import ( all_docs_from_collection, get_pi_id, key_value_pair_filter, - collection_str + collection_str, + _id_key ) from regolith.schemas import PROJECTUM_STATI, PROJECTUM_PAUSED_STATI, \ PROJECTUM_CANCELLED_STATI, PROJECTUM_FINISHED_STATI, PROJECTUM_ACTIVE_STATI diff --git a/regolith/helpers/l_progressreporthelper.py b/regolith/helpers/l_progressreporthelper.py index bc80191e8..c9115ff6a 100644 --- a/regolith/helpers/l_progressreporthelper.py +++ b/regolith/helpers/l_progressreporthelper.py @@ -5,11 +5,11 @@ """ from regolith.helpers.basehelper import SoutHelperBase -from regolith.fsclient import _id_key from regolith.tools import ( all_docs_from_collection, get_pi_id, key_value_pair_filter, + _id_key ) from gooey import GooeyParser diff --git a/regolith/helpers/l_projectahelper.py b/regolith/helpers/l_projectahelper.py index 52b3a9f67..b3539f663 100644 --- a/regolith/helpers/l_projectahelper.py +++ b/regolith/helpers/l_projectahelper.py @@ -7,14 +7,14 @@ import dateutil.parser as date_parser from regolith.helpers.basehelper import SoutHelperBase -from regolith.fsclient import _id_key from regolith.schemas import (PROJECTUM_ACTIVE_STATI, PROJECTUM_PAUSED_STATI, PROJECTUM_CANCELLED_STATI, PROJECTUM_FINISHED_STATI) from regolith.tools import ( all_docs_from_collection, get_pi_id, key_value_pair_filter, - collection_str + collection_str, + _id_key ) from gooey import GooeyParser diff --git a/regolith/helpers/l_todohelper.py b/regolith/helpers/l_todohelper.py index 978b5ef7c..6e29d0478 100644 --- a/regolith/helpers/l_todohelper.py +++ b/regolith/helpers/l_todohelper.py @@ -7,7 +7,6 @@ from regolith.dates import get_due_date from regolith.helpers.basehelper import SoutHelperBase -from regolith.fsclient import _id_key from regolith.schemas import ( TODO_STATI ) @@ -16,7 +15,8 @@ get_pi_id, document_by_value, print_task, - key_value_pair_filter + key_value_pair_filter, + _id_key ) from gooey import GooeyParser diff --git a/regolith/helpers/makeappointmentshelper.py b/regolith/helpers/makeappointmentshelper.py index 62cfefade..fec9b86ab 100644 --- a/regolith/helpers/makeappointmentshelper.py +++ b/regolith/helpers/makeappointmentshelper.py @@ -13,13 +13,14 @@ from datetime import timedelta, date from regolith.helpers.basehelper import SoutHelperBase -from regolith.fsclient import _id_key from regolith.tools import ( all_docs_from_collection, get_pi_id, is_fully_appointed, collect_appts, - grant_burn, group_member_employment_start_end, + grant_burn, + group_member_employment_start_end, + _id_key ) from regolith.dates import ( get_dates, diff --git a/regolith/helpers/u_contacthelper.py b/regolith/helpers/u_contacthelper.py index 6c6711d2f..0a92c9c6e 100644 --- a/regolith/helpers/u_contacthelper.py +++ b/regolith/helpers/u_contacthelper.py @@ -7,8 +7,7 @@ import uuid from regolith.helpers.basehelper import DbHelperBase -from regolith.fsclient import _id_key -from regolith.tools import all_docs_from_collection, fragment_retrieval +from regolith.tools import all_docs_from_collection, fragment_retrieval, _id_key from gooey import GooeyParser diff --git a/regolith/helpers/u_finishprumhelper.py b/regolith/helpers/u_finishprumhelper.py index d4445cbe3..66ecec775 100644 --- a/regolith/helpers/u_finishprumhelper.py +++ b/regolith/helpers/u_finishprumhelper.py @@ -1,8 +1,7 @@ """Helper for finishing prum in the projecta collection """ from regolith.helpers.basehelper import DbHelperBase -from regolith.fsclient import _id_key -from regolith.tools import all_docs_from_collection, fragment_retrieval +from regolith.tools import all_docs_from_collection, fragment_retrieval, _id_key import datetime as dt from dateutil import parser as date_parser from gooey import GooeyParser diff --git a/regolith/helpers/u_institutionshelper.py b/regolith/helpers/u_institutionshelper.py index a217d150c..ffca2af56 100644 --- a/regolith/helpers/u_institutionshelper.py +++ b/regolith/helpers/u_institutionshelper.py @@ -2,8 +2,7 @@ Helper for updating/adding to the projecta collection. """ from regolith.helpers.basehelper import DbHelperBase -from regolith.fsclient import _id_key -from regolith.tools import all_docs_from_collection, fragment_retrieval +from regolith.tools import all_docs_from_collection, fragment_retrieval, _id_key import uuid import datetime as dt from gooey import GooeyParser diff --git a/regolith/helpers/u_logurlhelper.py b/regolith/helpers/u_logurlhelper.py index 25b4b2650..f826df4ca 100644 --- a/regolith/helpers/u_logurlhelper.py +++ b/regolith/helpers/u_logurlhelper.py @@ -2,8 +2,7 @@ Log_urls are the google doc links to a projectum's Projectum Agenda Log """ from regolith.helpers.basehelper import DbHelperBase -from regolith.fsclient import _id_key -from regolith.tools import all_docs_from_collection, fragment_retrieval +from regolith.tools import all_docs_from_collection, fragment_retrieval, _id_key TARGET_COLL = "projecta" diff --git a/regolith/helpers/u_milestonehelper.py b/regolith/helpers/u_milestonehelper.py index 083b9f530..d4e43482f 100644 --- a/regolith/helpers/u_milestonehelper.py +++ b/regolith/helpers/u_milestonehelper.py @@ -9,8 +9,7 @@ from gooey import GooeyParser from regolith.helpers.basehelper import DbHelperBase -from regolith.fsclient import _id_key -from regolith.tools import all_docs_from_collection, fragment_retrieval +from regolith.tools import all_docs_from_collection, fragment_retrieval, _id_key from regolith.dates import get_due_date from regolith.schemas import PROJECTUM_ACTIVE_STATI, \ MILESTONE_TYPES, PROJECTUM_STATI diff --git a/regolith/helpers/u_todohelper.py b/regolith/helpers/u_todohelper.py index f810cab96..47319785e 100644 --- a/regolith/helpers/u_todohelper.py +++ b/regolith/helpers/u_todohelper.py @@ -7,14 +7,14 @@ import math from regolith.helpers.basehelper import DbHelperBase -from regolith.fsclient import _id_key from regolith.schemas import (TODO_STATI) from regolith.tools import ( all_docs_from_collection, get_pi_id, document_by_value, print_task, - key_value_pair_filter + key_value_pair_filter, + _id_key ) from gooey import GooeyParser diff --git a/regolith/helpers/v_meetingshelper.py b/regolith/helpers/v_meetingshelper.py index 9f30a2a4a..c581da4f5 100644 --- a/regolith/helpers/v_meetingshelper.py +++ b/regolith/helpers/v_meetingshelper.py @@ -3,11 +3,11 @@ import datetime as dt from regolith.helpers.basehelper import SoutHelperBase -from regolith.fsclient import _id_key from regolith.tools import ( all_docs_from_collection, get_pi_id, - validate_meeting + validate_meeting, + _id_key ) TARGET_COLL = "meetings" diff --git a/regolith/main.py b/regolith/main.py index 083640d37..a1cf4ef6c 100644 --- a/regolith/main.py +++ b/regolith/main.py @@ -342,12 +342,12 @@ def main(args=None): if rc.cmd in DISCONNECTED_COMMANDS: DISCONNECTED_COMMANDS[rc.cmd](rc) else: - dbs = None + colls = None if rc.cmd == 'build': - dbs = commands.build_db_check(rc) + colls = commands.build_db_check(rc) elif rc.cmd == 'helper': - dbs = commands.helper_db_check(rc) - with connect(rc, dbs=dbs) as rc.client: + colls = commands.helper_db_check(rc) + with connect(rc, colls=colls) as rc.client: CONNECTED_COMMANDS[rc.cmd](rc) diff --git a/regolith/mongoclient.py b/regolith/mongoclient.py index 7c7f27fb1..e05f1e9da 100644 --- a/regolith/mongoclient.py +++ b/regolith/mongoclient.py @@ -377,7 +377,7 @@ def load_database(self, db: dict): or coll in db["whitelist"] ]: col = mongodb[colname] - dbs[db['name']][colname] = load_mongo_col(col) + dbs[db['name']][colname] = col except OperationFailure as fail: print("Mongo's Error Message:" + str(fail) + "\n") print("The user does not have permission to access " + db['name'] + "\n\n") diff --git a/regolith/runcontrol.py b/regolith/runcontrol.py index 6a54866cb..16403787e 100644 --- a/regolith/runcontrol.py +++ b/regolith/runcontrol.py @@ -294,7 +294,7 @@ def connect_db(rc, colls=None): dbs: The databases in the form of a runcontrol client ''' - with connect(rc, dbs=colls) as rc.client: + with connect(rc, colls=colls) as rc.client: dbs = rc.client.dbs chained_db = rc.client.chained_db return chained_db, dbs diff --git a/regolith/tools.py b/regolith/tools.py index b2235cbf7..2dfa63ec8 100644 --- a/regolith/tools.py +++ b/regolith/tools.py @@ -42,6 +42,10 @@ ON_POSIX = os.name == "posix" +def _id_key(doc): + return doc["_id"] + + def dbdirname(db, rc): """Gets the database dir name.""" if db.get("local", False) is False: diff --git a/tests/test_broker.py b/tests/test_broker.py index e851c9b00..7229bf924 100644 --- a/tests/test_broker.py +++ b/tests/test_broker.py @@ -13,6 +13,6 @@ def test_round_trip(make_db, tmpdir): os.chdir(repo) subprocess.check_call(["touch", "myfile.tex"], cwd=tmpdir) db = load_db() - db.add_file(db["projects"]["Cyclus"], "myfile", os.path.join(tmpdir, "myfile.tex")) - ret = db.get_file_path(db["projects"]["Cyclus"], "myfile") + db.add_file(db["test"]["projects"]["Cyclus"], "myfile", os.path.join(tmpdir, "myfile.tex")) + ret = db.get_file_path(db["test"]["projects"]["Cyclus"], "myfile") assert ret == os.path.join(repo, "myfile.tex") diff --git a/tests/test_builders.py b/tests/test_builders.py index c47d1a764..531b2ca09 100644 --- a/tests/test_builders.py +++ b/tests/test_builders.py @@ -62,9 +62,9 @@ def prep_figure(): f.write("hello world") # load the db and register the file db = load_db() - print(db.get_file_path(db["groups"]["ergs"], "hello")) - if not db.get_file_path(db["groups"]["ergs"], "hello"): - db.add_file(db["groups"]["ergs"], "hello", "fig/hello.txt") + print(db.get_file_path(db["test"]["groups"]["ergs"], "hello")) + if not db.get_file_path(db["test"]["groups"]["ergs"], "hello"): + db.add_file(db["test"]["groups"]["ergs"], "hello", "fig/hello.txt") @pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") diff --git a/tests/test_chain_db.py b/tests/test_chain_db.py index 862b5aa0a..e576198cb 100644 --- a/tests/test_chain_db.py +++ b/tests/test_chain_db.py @@ -1,11 +1,11 @@ -from regolith.chained_db import ChainDB +from regolith.chained_db import ChainDocument def test_dddi(): a = {"a": {"a": {"a": 1}}} - z = ChainDB(a) - assert isinstance(z["a"], ChainDB) - assert isinstance(z["a"]["a"], ChainDB) + z = ChainDocument(a) + assert isinstance(z["a"], ChainDocument) + assert isinstance(z["a"]["a"], ChainDocument) assert isinstance(z["a"]["a"]["a"], int) assert z["a"]["a"]["a"] + 1 == 2 @@ -13,9 +13,9 @@ def test_dddi(): def test_second_mapping(): m1 = {"a": {"m": {"x": 0}}} m2 = {"a": {"m": {"y": 1}}} - z = ChainDB(m1) + z = ChainDocument(m1) z.maps.append(m2) - assert isinstance(z["a"], ChainDB) + assert isinstance(z["a"], ChainDocument) assert isinstance(z["a"]["m"].maps, list) assert z["a"]["m"]["y"] == 1 @@ -23,9 +23,9 @@ def test_second_mapping(): def test_double_mapping(): m1 = {"a": {"m": {"y": 0}}} m2 = {"a": {"m": {"y": 1}}} - z = ChainDB(m1) + z = ChainDocument(m1) z.maps.append(m2) - assert isinstance(z["a"], ChainDB) + assert isinstance(z["a"], ChainDocument) assert isinstance(z["a"]["m"].maps, list) assert isinstance(z["a"]["m"]["y"], int) assert z["a"]["m"]["y"] == 1 @@ -34,9 +34,9 @@ def test_double_mapping(): def test_list_mapping(): m1 = {"a": {"m": "x"}} m2 = {"a": {"m": "y"}} - z = ChainDB(m1) + z = ChainDocument(m1) z.maps.append(m2) - assert isinstance(z["a"], ChainDB) + assert isinstance(z["a"], ChainDocument) assert isinstance(z["a"]["m"], str) assert z["a"]["m"] == "y" @@ -44,9 +44,9 @@ def test_list_mapping(): def test_mixed_mapping(): m1 = {"a": {"m": {"y": 1}}} m2 = {"a": {"m": 1}} - z = ChainDB(m1) + z = ChainDocument(m1) z.maps.append(m2) - assert isinstance(z["a"], ChainDB) + assert isinstance(z["a"], ChainDocument) assert isinstance(z["a"]["m"], int) assert z["a"]["m"] == 1 @@ -54,9 +54,9 @@ def test_mixed_mapping(): def test_exactness(): d = {"y": 1} m1 = {"a": {"m": d}} - z = ChainDB(m1) - assert isinstance(z["a"], ChainDB) - assert isinstance(z["a"]["m"], ChainDB) + z = ChainDocument(m1) + assert isinstance(z["a"], ChainDocument) + assert isinstance(z["a"]["m"], ChainDocument) assert isinstance(z["a"]["m"].maps[0], dict) assert d is z["a"]["m"].maps[0] @@ -64,11 +64,11 @@ def test_exactness(): def test_exactness_setting(): d = {"y": 1} m1 = {"a": {"m": d}} - z = ChainDB(m1) + z = ChainDocument(m1) e = {"z": 2} z["a"]["m"] = e - assert isinstance(z["a"], ChainDB) - assert isinstance(z["a"]["m"], ChainDB) + assert isinstance(z["a"], ChainDocument) + assert isinstance(z["a"]["m"], ChainDocument) assert isinstance(z["a"]["m"].maps[0], dict) assert e is z["a"]["m"].maps[0] @@ -78,11 +78,11 @@ def test_exactness_setting_multi(): e = "b" m1 = {"a": {"m": d}} m2 = {"a": {"m": e}} - z = ChainDB(m1) + z = ChainDocument(m1) z.maps.append(m2) g = ("c",) z["a"]["m"] = g - assert isinstance(z["a"], ChainDB) + assert isinstance(z["a"], ChainDocument) assert isinstance(z["a"]["m"], tuple) assert isinstance(z["a"].maps[0], dict) assert g is z["a"].maps[1]["m"] @@ -96,11 +96,11 @@ def test_exactness_setting_multi2(): ee = [5, 6] m1 = {"a": {"m": d}} m2 = {"a": {"m": e, "mm": ee}} - z = ChainDB(m1) + z = ChainDocument(m1) z.maps.append(m2) g = [-1, -2] z["a"]["mm"] = g - assert isinstance(z["a"], ChainDB) + assert isinstance(z["a"], ChainDocument) assert isinstance(z["a"]["m"], list) assert isinstance(z["a"].maps[0], dict) assert g is z["a"].maps[1]["mm"] @@ -113,11 +113,11 @@ def test_exactness_setting_multi_novel(): e = [3, 4] m1 = {"a": {"m": d}} m2 = {"a": {"m": e}} - z = ChainDB(m1) + z = ChainDocument(m1) z.maps.append(m2) g = [-1, -2] z["a"]["mm"] = g - assert isinstance(z["a"], ChainDB) + assert isinstance(z["a"], ChainDocument) assert isinstance(z["a"]["m"], list) assert isinstance(z["a"].maps[0], dict) assert g is z["a"].maps[0]["mm"] @@ -130,9 +130,9 @@ def test_dicts_in_lists(): t = c + d m1 = {"a": {"b": c}} m2 = {"a": {"b": d}} - z = ChainDB(m1) + z = ChainDocument(m1) z.maps.append(m2) - assert isinstance(z["a"], ChainDB) + assert isinstance(z["a"], ChainDocument) assert isinstance(z["a"]["b"], list) assert z["a"]["b"] == t assert c[0] is z["a"]["b"][0] @@ -146,7 +146,7 @@ def test_dicts_in_lists_mutation(): d = [{"o": 3}, {"p": 4}] m1 = {"a": {"b": c}} m2 = {"a": {"b": d}} - z = ChainDB(m1) + z = ChainDocument(m1) z.maps.append(m2) append_list = z["a"]["b"] append_list.append({"hi": "world"}) diff --git a/tests/test_runcontrol.py b/tests/test_runcontrol.py index b5c0af461..9f8f1d629 100644 --- a/tests/test_runcontrol.py +++ b/tests/test_runcontrol.py @@ -16,5 +16,21 @@ def test_connect_db(make_db): expected_dbs = rc.client.dbs expected_chdb = rc.client.chained_db chained_db, dbs = connect_db(rc) - assert chained_db == expected_chdb - assert dbs == expected_dbs + for coll in chained_db.keys(): + if coll in expected_chdb: + for k in chained_db[coll].keys(): + if k in expected_chdb[coll]: + assert chained_db[coll][k] == expected_chdb[coll][k] + else: + assert False + else: + assert False + for coll in dbs.keys(): + if coll in expected_dbs: + for k in dbs[coll].keys(): + if k in expected_dbs[coll]: + assert dbs[coll][k] == expected_dbs[coll][k] + else: + assert False + else: + assert False