From b4721e92d14b8bef61e37b641787a3e8d802dfa8 Mon Sep 17 00:00:00 2001 From: Sondov Engen Date: Mon, 26 Jan 2026 17:17:07 +0100 Subject: [PATCH 1/2] visparse: add `--load` support to cli This allows us to `--load` folders that have been `--dumped` --- novem/cli/common.py | 15 ++++++++++++++ novem/cli/setup.py | 10 ++++++++++ novem/job/__init__.py | 42 +++++++++++++++++++++++++++++++++++++++ novem/vis/__init__.py | 46 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 113 insertions(+) diff --git a/novem/cli/common.py b/novem/cli/common.py index 1014532..25b8d08 100644 --- a/novem/cli/common.py +++ b/novem/cli/common.py @@ -131,6 +131,14 @@ def __call__(self, args: Dict[str, Any]) -> None: vis.api_dump(outpath=path) return + # --load: load folder structure into API + if "load" in args and args["load"]: + path = args["load"] + + print(f'Loading api tree structure from "{path}"') + vis.api_load(inpath=path) + return + # if we detect a tree query then we'll discard all other IO if "tree" in args and args["tree"] != -1: path = args["tree"] @@ -340,6 +348,13 @@ def job(args: Dict[str, Any]) -> None: j.api_dump(outpath=path) return + # --load: load folder structure into API + if "load" in args and args["load"]: + path = args["load"] + print(f'Loading api tree structure from "{path}"') + j.api_load(inpath=path) + return + # --tree: print API tree structure if "tree" in args and args["tree"] != -1: path = args["tree"] diff --git a/novem/cli/setup.py b/novem/cli/setup.py index a025993..9d88b35 100644 --- a/novem/cli/setup.py +++ b/novem/cli/setup.py @@ -83,6 +83,16 @@ def setup(raw_args: Any = None) -> Tuple[Any, Dict[str, str]]: help=ap.SUPPRESS, ) + parser.add_argument( + "--load", + metavar=("IN_PATH"), + dest="load", + action="store", + required=False, + default=None, + help=ap.SUPPRESS, + ) + parser.add_argument( "--version", dest="version", diff --git a/novem/job/__init__.py b/novem/job/__init__.py index 6ab661d..ad952d5 100644 --- a/novem/job/__init__.py +++ b/novem/job/__init__.py @@ -364,6 +364,48 @@ def rec_tree(path: str) -> None: # start recursion rec_tree("/") + def api_load(self, inpath: str) -> None: + """ + Load a dumped folder structure back into the API. + Walks the folder and for each file: PUT to create, then POST content. + """ + + qpath = f"{self._api_root}jobs/{self.id}" + + def load_tree(local_path: str, api_path: str) -> None: + full_local = os.path.join(inpath, local_path.lstrip("/")) if local_path else inpath + + if os.path.isfile(full_local): + # Read file content + with open(full_local, "r") as f: + content = f.read() + + full_api = f"{qpath}{api_path}" + + # Try PUT first to create the resource + r = self._session.put(full_api) + put_status = r.status_code + + # POST the content + r = self._session.post( + full_api, + headers={"Content-type": "text/plain"}, + data=content.encode("utf-8"), + ) + print(f"Loaded file: {api_path} (PUT: {put_status}, POST: {r.status_code}, {len(content)} bytes)") + + elif os.path.isdir(full_local): + print(f"Processing dir: {api_path or '/'}") + + # Iterate over directory contents + for entry in sorted(os.listdir(full_local)): + entry_local = os.path.join(local_path, entry) if local_path else entry + entry_api = f"{api_path}/{entry}" + load_tree(entry_local, entry_api) + + # Start loading from root + load_tree("", "") + def api_tree(self, colors: bool = False, relpath: str = "/") -> str: """ Iterate over the current job and print a "pretty" ascii tree diff --git a/novem/vis/__init__.py b/novem/vis/__init__.py index 480d963..8a7708c 100644 --- a/novem/vis/__init__.py +++ b/novem/vis/__init__.py @@ -98,6 +98,52 @@ def rec_tree(path: str) -> None: # start recurison rec_tree("") + def api_load(self, inpath: str) -> None: + """ + Load a dumped folder structure back into the API. + Walks the folder and for each file: PUT to create, then POST content. + """ + + qpath = f"{self._api_root}vis/{self._vispath}/{self.id}" + + if self.user: + print(f"you cannot modify another users {self._vispath}") + return + + def load_tree(local_path: str, api_path: str) -> None: + full_local = os.path.join(inpath, local_path.lstrip("/")) if local_path else inpath + + if os.path.isfile(full_local): + # Read file content + with open(full_local, "r") as f: + content = f.read() + + full_api = f"{qpath}{api_path}" + + # Try PUT first to create the resource + r = self._session.put(full_api) + put_status = r.status_code + + # POST the content + r = self._session.post( + full_api, + headers={"Content-type": "text/plain"}, + data=content.encode("utf-8"), + ) + print(f"Loaded file: {api_path} (PUT: {put_status}, POST: {r.status_code}, {len(content)} bytes)") + + elif os.path.isdir(full_local): + print(f"Processing dir: {api_path or '/'}") + + # Iterate over directory contents + for entry in sorted(os.listdir(full_local)): + entry_local = os.path.join(local_path, entry) if local_path else entry + entry_api = f"{api_path}/{entry}" + load_tree(entry_local, entry_api) + + # Start loading from root + load_tree("", "") + def api_tree(self, colors: bool = False, relpath: str = "/") -> str: """ Iterate over the current id and print a "pretty" ascii tree From a9071ea4acf0a9d50674bf1bad4d6a641d8f0756 Mon Sep 17 00:00:00 2001 From: Sondov Engen Date: Mon, 26 Jan 2026 22:08:05 +0100 Subject: [PATCH 2/2] cli: don't include default files and folders in `--dump` --- novem/job/__init__.py | 41 +++++++++++++++++++++++++++++------- novem/vis/__init__.py | 49 +++++++++++++++++++++++++++++++++---------- 2 files changed, 72 insertions(+), 18 deletions(-) diff --git a/novem/job/__init__.py b/novem/job/__init__.py index ad952d5..63c258f 100644 --- a/novem/job/__init__.py +++ b/novem/job/__init__.py @@ -346,20 +346,47 @@ def rec_tree(path: str) -> None: # if i am a file, write to disc if tp == "file": + # skip files with default values + if headers.get("x-nvm-default", "").lower() == "true": + print(f"Skipping default: {fp}") + return None + # ensure parent directory exists before writing + parent_dir = os.path.dirname(fp) + if parent_dir and not os.path.exists(parent_dir): + os.makedirs(parent_dir, exist_ok=True) + print(f"Creating folder: {parent_dir}") with open(fp, "w") as f: f.write(req.text) print(f"Writing file: {fp}") return None - # if I am a folder, make a folder and recurse - os.makedirs(fp, exist_ok=True) - print(f"Creating folder: {fp}") - + # if I am a folder, recurse without creating yet nodes: List[Dict[str, str]] = req.json() - # Recurse relevant structure - for r in [x for x in nodes if x["type"] not in ["system_file", "system_dir"]]: - rec_tree(f'{path}/{r["name"]}') + # Recurse relevant structure (skip system entries and read-only files) + for r in nodes: + if r["type"] in ["system_file", "system_dir"]: + continue + child_path = f'{path}/{r["name"]}' + child_fp = f"{outpath}{child_path}" + + # /shared/ and /tags/ are special markers - create empty files from listing + if r["type"] in ["file", "link"] and ( + child_path.startswith("/shared/") or child_path.startswith("/tags/") + ): + parent_dir = os.path.dirname(child_fp) + if parent_dir and not os.path.exists(parent_dir): + os.makedirs(parent_dir, exist_ok=True) + print(f"Creating folder: {parent_dir}") + with open(child_fp, "w") as f: + f.write("") + print(f"Writing file: {child_fp}") + continue + + # skip read-only files/links + if r["type"] in ["file", "link"] and "w" not in r.get("permissions", []): + continue + rec_tree(child_path) # start recursion rec_tree("/") diff --git a/novem/vis/__init__.py b/novem/vis/__init__.py index 8a7708c..39deed3 100644 --- a/novem/vis/__init__.py +++ b/novem/vis/__init__.py @@ -80,20 +80,47 @@ def rec_tree(path: str) -> None: # if i am a file, write to disc if tp == "file": + # skip files with default values + if headers.get("x-nvm-default", "").lower() == "true": + print(f"Skipping default: {fp}") + return None + # ensure parent directory exists before writing + parent_dir = os.path.dirname(fp) + if parent_dir and not os.path.exists(parent_dir): + os.makedirs(parent_dir, exist_ok=True) + print(f"Creating folder: {parent_dir}") with open(fp, "w") as f: f.write(req.text) print(f"Writing file: {fp}") return None - # if I am a folder, make a folder and recurse - os.makedirs(fp, exist_ok=True) - print(f"Creating folder: {fp}") - + # if I am a folder, recurse without creating yet nodes: List[Dict[str, str]] = req.json() - # Recurse relevant structure - for r in [x for x in nodes if x["type"] not in ["system_file", "system_dir"]]: - rec_tree(f'{path}/{r["name"]}') + # Recurse relevant structure (skip system entries and read-only files) + for r in nodes: + if r["type"] in ["system_file", "system_dir"]: + continue + child_path = f'{path}/{r["name"]}' + child_fp = f"{outpath}{child_path}" + + # /shared/ and /tags/ are special markers - create empty files from listing + if r["type"] in ["file", "link"] and ( + child_path.startswith("/shared/") or child_path.startswith("/tags/") + ): + parent_dir = os.path.dirname(child_fp) + if parent_dir and not os.path.exists(parent_dir): + os.makedirs(parent_dir, exist_ok=True) + print(f"Creating folder: {parent_dir}") + with open(child_fp, "w") as f: + f.write("") + print(f"Writing file: {child_fp}") + continue + + # skip read-only files/links + if r["type"] in ["file", "link"] and "w" not in r.get("permissions", []): + continue + rec_tree(child_path) # start recurison rec_tree("") @@ -107,7 +134,7 @@ def api_load(self, inpath: str) -> None: qpath = f"{self._api_root}vis/{self._vispath}/{self.id}" if self.user: - print(f"you cannot modify another users {self._vispath}") + print(f"You cannot modify another user's {self._vispath}") return def load_tree(local_path: str, api_path: str) -> None: @@ -317,7 +344,7 @@ def api_delete(self, relpath: str) -> None: value: the value to write to the file """ if self.user: - print(f"you cannot modify another users {self._vispath}") + print(f"You cannot modify another user's {self._vispath}") return path = f"{self._api_root}vis/{self._vispath}/{self.id}{relpath}" @@ -352,7 +379,7 @@ def api_create(self, relpath: str) -> None: value: the value to write to the file """ if self.user: - print(f"you cannot modify another users {self._vispath}") + print(f"You cannot modify another user's {self._vispath}") return path = f"{self._api_root}vis/{self._vispath}/{self.id}{relpath}" @@ -392,7 +419,7 @@ def api_write(self, relpath: str, value: str) -> None: value: the value to write to the file """ if self.user: - print(f"you cannot modify another users {self._vispath}") + print(f"You cannot modify another user's {self._vispath}") return path = f"{self._api_root}vis/{self._vispath}/{self.id}{relpath}"