Speed up directory tree generation (#736)

# Motivation

<!-- Why is this change necessary? -->

# Content

<!-- Please include a summary of the change -->

# Testing

<!-- How was the change tested? -->

# Please check the following before marking your PR as ready for review

- [ ] I have added tests for my changes
- [ ] I have updated the documentation or added new documentation as
needed

Author: EdwardJXLi

src/codegen/git/repo_operator/repo_operator.py

@@ -616,6 +616,7 @@ def iter_files(
         subdirs: list[str] | None = None,
         extensions: list[str] | None = None,
         ignore_list: list[str] | None = None,
+        skip_content: bool = False,
     ) -> Generator[tuple[str, str]]:
         """Iterates over all files in the codebase, yielding the filepath and its content.
 
@@ -642,8 +643,11 @@ def iter_files(
 
             if extensions is None or any(filepath.endswith(e) for e in extensions):
                 try:
-                    content = self.get_file(filepath)
-                    yield rel_filepath, content
+                    if not skip_content:
+                        content = self.get_file(filepath)
+                        yield rel_filepath, content
+                    else:
+                        yield rel_filepath, ""
                 except Exception as e:
                     logger.warning(f"Error reading file {filepath}: {e}")
 

src/codegen/sdk/codebase/codebase_context.py

@@ -365,7 +365,11 @@ def build_directory_tree(self) -> None:
         # Reset and rebuild the directory tree
         self.directories = dict()
 
-        for file_path, _ in self.projects[0].repo_operator.iter_files(subdirs=self.projects[0].subdirectories, ignore_list=GLOBAL_FILE_IGNORE_LIST):
+        for file_path, _ in self.projects[0].repo_operator.iter_files(
+            subdirs=self.projects[0].subdirectories,
+            ignore_list=GLOBAL_FILE_IGNORE_LIST,
+            skip_content=True,
+        ):
             file_path = Path(file_path)
             directory = self.get_directory(file_path.parent, create_on_missing=True)
             directory._add_file(file_path.name)
@@ -510,7 +514,6 @@ def _process_diff_files(self, files_to_sync: Mapping[SyncType, list[Path]], incr
 
         # Step 6: Build directory tree
         logger.info("> Building directory tree")
-        files = [f for f in sort_editables(self.get_nodes(NodeType.FILE), alphabetical=True, dedupe=False)]
         self.build_directory_tree()
 
         # Step 7: Build configs