From 6a4d063712f76c98068ba3a5a846f2daf7a427ad Mon Sep 17 00:00:00 2001 From: lijialin03 Date: Thu, 4 Dec 2025 03:12:15 +0000 Subject: [PATCH] fix:support np.array in traverse --- .github/workflows/tests.yml | 9 +++++++++ padiff/abstracts/hooks/hook.py | 3 +++ padiff/tools/dump.py | 10 +++++++++- padiff/utils/utils.py | 4 ++++ 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1a665cc..979d281 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,11 +26,20 @@ jobs: with: python-version: '3.10' cache: 'pip' + - name: Clean system disk space + run: | + sudo apt-get clean + sudo rm -rf /var/lib/apt/lists/* + echo "Disk space after cleanup:" + df -h / - name: Install dependencies run: | python -m pip install --upgrade pip pip install -e .[full] make install + pip cache purge + sudo apt-get clean + sudo rm -rf /var/lib/apt/lists/* - name: Run tests run: make test - name: Upload coverage report diff --git a/padiff/abstracts/hooks/hook.py b/padiff/abstracts/hooks/hook.py index 3291108..ec4dd32 100644 --- a/padiff/abstracts/hooks/hook.py +++ b/padiff/abstracts/hooks/hook.py @@ -139,6 +139,9 @@ def info_hook(model, input, output, net_id): # if this api is not processing tensors, do not create report if output is None or all([not isinstance(x, (paddle.Tensor, torch.Tensor)) for x in flatten(output)]): + logger.warning_once( + f"All outputs of {model.__class__.__name__} are not tensors. Skip capturing these outputs." + ) return None # if an api under black_list_recursively, do not create report diff --git a/padiff/tools/dump.py b/padiff/tools/dump.py index 4a83314..66a95b1 100644 --- a/padiff/tools/dump.py +++ b/padiff/tools/dump.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from collections import defaultdict import json import os import sys @@ -130,6 +131,8 @@ def dump_report_node(wrap_node, tensor_dumper): def dump_param_prototype(model, dump_fn, file_path): + skiped_layers = defaultdict(list) + def dump_param_with_fn(model, fn, target_models): param_info = { "name": model.class_name, @@ -150,7 +153,7 @@ def dump_param_with_fn(model, fn, target_models): if buffer_name not in params_found: fn(buffer_name, buffer, param_info) else: - logger.debug(f"Layer {model.class_name} ({model.route}) is NOT in target_models. Skipping.") + skiped_layers[model.class_name].append(model.route) for name, child in model.named_children(): param_info["children"].append(dump_param_with_fn(child, fn, target_models)) @@ -159,6 +162,11 @@ def dump_param_with_fn(model, fn, target_models): target_models = [layer.model for layer in model.marker.traversal_for_assign_weight()] param_info = dump_param_with_fn(model, dump_fn, target_models) + logger.debug_once("Params dump SKIPPED: Some layers have no available parameters(like weights).\n") + for model_name, routes in skiped_layers.items(): + routes_str = "\n".join([f" {route}" for route in routes]) + logger.debug(f"Params dump SKIPPED: {model_name}.\nIncluded routes:\n{routes_str}") + model_info = { "model_name": model.name, "framework": model.framework, diff --git a/padiff/utils/utils.py b/padiff/utils/utils.py index 413ab13..83f7c98 100644 --- a/padiff/utils/utils.py +++ b/padiff/utils/utils.py @@ -144,6 +144,10 @@ def traverse(structure, on_leaf, on_container=None): if isinstance(structure, (paddle.Tensor, torch.Tensor)): return on_leaf(structure) + # numpy + if isinstance(structure, np.ndarray): + return on_leaf(structure) + # namedtuple if hasattr(structure, "_fields"): result = type(structure)(