Add -tools build command to modularize internal build processing.

.python-version

@@ -1 +1 @@
-3.8
+3.9

aws_doc_sdk_examples_tools/builder.py

@@ -0,0 +1,62 @@
+from dataclasses import dataclass, field
+import logging
+from pathlib import Path
+
+from aws_doc_sdk_examples_tools.doc_gen import DocGen
+from aws_doc_sdk_examples_tools.fs import Fs, PathFs
+from aws_doc_sdk_examples_tools.snippets import write_snippets
+
+logging.basicConfig(level=logging.INFO)
+
+logger = logging.getLogger(Path(__file__).name)
+
+
+@dataclass
+class Builder:
+    root: Path
+    dest: Path
+    doc_gen_folder = ".doc_gen"
+    snippet_files_folder = "snippet_files"
+    fs: Fs = field(default_factory=PathFs)
+    _doc_gen: DocGen = field(init=False)
+
+    def __post_init__(self):
+        self._doc_gen = DocGen.from_root(self.root)
+
+    def copy_doc_gen(self):
+        tmp_mirror_doc_gen = self.root / self.doc_gen_folder
+        mirror_doc_gen = self.dest / self.doc_gen_folder
+        logger.info(f"Moving cloned files into package from {tmp_mirror_doc_gen} to {mirror_doc_gen}")
+        try:
+            self.fs.copytree(tmp_mirror_doc_gen, mirror_doc_gen)
+        except Exception as e:
+            logger.error(f"Failed copy directory {tmp_mirror_doc_gen} to {mirror_doc_gen}\n{e}")
+            logger.error(e)
+            raise
+
+    def write_snippets(self):
+        write_snippets(self.dest / self.doc_gen_folder / self.snippet_files_folder, self._doc_gen.snippets, self.fs)
+
+    def run(self):
+        logger.debug("Copying docgen files...")
+        self.copy_doc_gen()
+        logger.debug("Collecting snippets...")
+        self._doc_gen.collect_snippets()
+        logger.debug("Writing snippets...")
+        self.write_snippets()
+
+
+def main():
+    from argparse import ArgumentParser
+    argparse = ArgumentParser()
+    argparse.add_argument('--root', type=Path, default=Path())
+    argparse.add_argument('--dest', type=Path)
+    # Load a config from somewhere to get doc_gen_folder and snippets_files_folder
+    args = argparse.parse_args()
+
+    builder = Builder(args.root, args.dest)
+    builder.run()
+
+
+if __name__ == "__main__":
+    main()

aws_doc_sdk_examples_tools/fs.py

@@ -3,6 +3,7 @@
 from fnmatch import fnmatch
 from os import listdir
 from pathlib import Path
+import shutil
 from stat import S_ISREG
 from typing import Dict, Generator, List
 
@@ -47,6 +48,10 @@ def mkdir(self, path: Path):
     def list(self, path: Path) -> List[Path]:
         pass
 
+    @abstractmethod
+    def copytree(self, source: Path, dest: Path):
+        pass
+
 
 class PathFs(Fs):
     def glob(self, path: Path, glob: str) -> Generator[Path, None, None]:
@@ -61,6 +66,7 @@ def readlines(self, path: Path, encoding: str = "utf-8") -> List[str]:
             return file.readlines()
 
     def write(self, path: Path, content: str):
+        self.mkdir(path.parent)
         with path.open("w", encoding="utf-8") as file:
             file.write(content)
 
@@ -79,6 +85,9 @@ def list(self, path: Path) -> List[Path]:
             return []
         return [path / name for name in listdir(path)]
 
+    def copytree(self, source: Path, dest: Path):
+        shutil.copytree(source, dest, dirs_exist_ok=True)
+
 
 class RecordFs(Fs):
     def __init__(self, fs: Dict[Path, str]):
@@ -137,5 +146,10 @@ def list(self, path: Path) -> List[Path]:
 
         return sorted(children)
 
+    def copytree(self, source: Path, dest: Path):
+        for child in self.list(source):
+            path = child.relative_to(dest).absolute()
+            self.fs[path] = self.fs[child]
+
 
 fs = PathFs()

aws_doc_sdk_examples_tools/snippets.py

@@ -17,6 +17,7 @@
     ValidationConfig,
 )
 from . import validator_config
+from aws_doc_sdk_examples_tools import fs
 
 SNIPPET_START = "snippet-start:["
 SNIPPET_END = "snippet-end:["
@@ -309,16 +310,15 @@ def validate_snippets(
         verify_no_secret_keys(snippet.code, root / snippet.file, validation, errors)
 
 
-def write_snippets(root: Path, snippets: Dict[str, Snippet], check: bool = False):
+def write_snippets(root: Path, snippets: Dict[str, Snippet], check: bool = False, fs: Fs = fs.fs):
     errors = MetadataErrors()
     for tag in snippets:
         name = root / f"{tag}.txt"
-        if check and name.exists():
+        if check and fs.stat(name).exists:
             errors.append(SnippetAlreadyWritten(file=name))
         else:
             try:
-                with open(name, "w", encoding="utf-8") as file:
-                    file.write(snippets[tag].code)
+                fs.write(name, snippets[tag].code)
             except Exception as error:
                 errors.append(SnippetWriteError(file=name, error=error))
     return errors