From efa02d1fea5ce5b6c5e140673f59b3b1b4cfb06a Mon Sep 17 00:00:00 2001 From: "Druzhkov, Pavel" Date: Thu, 7 Oct 2021 15:42:28 +0300 Subject: [PATCH 1/9] up --- .../apis/ote/apis/detection/configuration.py | 15 ++++++ .../ote/apis/detection/configuration.yaml | 20 ++++++++ .../apis/ote/apis/detection/inference_task.py | 51 +++++++++++++++---- mmdet/apis/ote/apis/detection/ote_utils.py | 36 +++++++++++++ 4 files changed, 112 insertions(+), 10 deletions(-) diff --git a/mmdet/apis/ote/apis/detection/configuration.py b/mmdet/apis/ote/apis/detection/configuration.py index f7e85d64cd3..15b803f056f 100644 --- a/mmdet/apis/ote/apis/detection/configuration.py +++ b/mmdet/apis/ote/apis/detection/configuration.py @@ -145,7 +145,22 @@ class __POTParameter(ParameterGroup): description="Quantization preset that defines quantization scheme", editable=True, visible_in_ui=True) + + @attrs + class __DebugParameters(ParameterGroup): + header = string_attribute("Debugging Parameters") + description = header + + enable_debug_dump = configurable_boolean( + default_value=True, + header="Enable data dumps for debugging", + description="Enable data dumps for debugging", + affects_outcome_of=ModelLifecycle.NONE + ) + + learning_parameters = add_parameter_group(__LearningParameters) postprocessing = add_parameter_group(__Postprocessing) nncf_optimization = add_parameter_group(__NNCFOptimization) pot_parameters = add_parameter_group(__POTParameter) + debug_parameters = add_parameter_group(__DebugParameters) diff --git a/mmdet/apis/ote/apis/detection/configuration.yaml b/mmdet/apis/ote/apis/detection/configuration.yaml index 66f21b016ec..b42b66d7a30 100644 --- a/mmdet/apis/ote/apis/detection/configuration.yaml +++ b/mmdet/apis/ote/apis/detection/configuration.yaml @@ -1,3 +1,23 @@ +debug_parameters: + description: Debugging Parameters + enable_debug_dump: + affects_outcome_of: NONE + default_value: true + description: Enable data dumps for debugging + editable: true + header: Enable data dumps for debugging + type: BOOLEAN + ui_rules: + action: DISABLE_EDITING + operator: AND + rules: [] + type: UI_RULES + value: true + visible_in_ui: true + warning: null + header: Debugging Parameters + type: PARAMETER_GROUP + visible_in_ui: true description: Configuration for an object detection task header: Configuration for an object detection task learning_parameters: diff --git a/mmdet/apis/ote/apis/detection/inference_task.py b/mmdet/apis/ote/apis/detection/inference_task.py index 814d0586885..ad49390ab48 100644 --- a/mmdet/apis/ote/apis/detection/inference_task.py +++ b/mmdet/apis/ote/apis/detection/inference_task.py @@ -16,6 +16,7 @@ import io import logging import os +import pickle import shutil import tempfile import warnings @@ -52,11 +53,13 @@ from mmdet.parallel import MMDataCPU logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) class OTEDetectionInferenceTask(IInferenceTask, IExportTask, IEvaluationTask, IUnload): _task_environment: TaskEnvironment + _debug_dump_file_path: str = '/tmp/debug_dump.pkl' def __init__(self, task_environment: TaskEnvironment): """" @@ -68,8 +71,6 @@ def __init__(self, task_environment: TaskEnvironment): self._scratch_space = tempfile.mkdtemp(prefix="ote-det-scratch-") logger.info(f"Scratch space created at {self._scratch_space}") - self._hyperparams = hyperparams = task_environment.get_hyper_parameters(OTEDetectionConfig) - self._model_name = task_environment.model_template.name self._labels = task_environment.get_labels(False) @@ -80,7 +81,7 @@ def __init__(self, task_environment: TaskEnvironment): config_file_path = os.path.join(self._base_dir, "model.py") self._config = Config.fromfile(config_file_path) patch_config(self._config, self._scratch_space, self._labels, random_seed=42) - set_hyperparams(self._config, hyperparams) + set_hyperparams(self._config, self._hyperparams) # Create and initialize PyTorch model. self._model = self._load_model(task_environment.model) @@ -91,6 +92,11 @@ def __init__(self, task_environment: TaskEnvironment): self._should_stop = False + @property + def _hyperparams(self): + return self._task_environment.get_hyper_parameters(OTEDetectionConfig) + + def _load_model(self, model: ModelEntity): if model is not None: # If a model has been trained and saved for the task already, create empty model and load weights here @@ -142,9 +148,39 @@ def _create_model(config: Config, from_scratch: bool = False): model = build_detector(model_cfg) return model + def _dump_model(self): + hyperparams_str = ids_to_strings(cfg_helper.convert(self._hyperparams, dict, enum_to_str=True)) + labels = {label.name: label.color.rgb_tuple for label in self._labels} + weights = self._model.state_dict() + config = self._config + return { + 'hyperparams': hyperparams_str, + 'labels': labels, + 'weights': weights, + 'config': config + } def infer(self, dataset: DatasetEntity, inference_parameters: Optional[InferenceParameters] = None) -> DatasetEntity: """ Analyzes a dataset using the latest inference model. """ + + if self._hyperparams.debug_parameters.enable_debug_dump: + from mmdet.apis.ote.apis.detection.ote_utils import dump_dataset + + class_name = self.__class__.__name__ + func_name = 'infer' + dump_dict = { + 'class_name': class_name, + 'entrypoint': func_name, + 'model': self._dump_model(), + 'arguments': { + 'dataset': dump_dataset(dataset), + # 'inference_parameters': inference_parameters + } + } + logger.warning(f'Saving debug dump for {class_name}.{func_name} call to {self._debug_dump_file_path}') + with open(self._debug_dump_file_path, 'ab') as fp: + pickle.dump(dump_dict, fp) + set_hyperparams(self._config, self._hyperparams) if inference_parameters is not None: @@ -234,9 +270,8 @@ def evaluate(self, output_result_set: ResultSetEntity, evaluation_metric: Optional[str] = None): """ Computes performance on a resultset """ - params = self._hyperparams - result_based_confidence_threshold = params.postprocessing.result_based_confidence_threshold + result_based_confidence_threshold = self._hyperparams.postprocessing.result_based_confidence_threshold logger.info('Computing F-measure' + (' with auto threshold adjustment' if result_based_confidence_threshold else '')) f_measure_metrics = MetricsHelper.compute_f_measure(output_result_set, @@ -250,11 +285,9 @@ def evaluate(self, best_confidence_threshold = f_measure_metrics.best_confidence_threshold.value if best_confidence_threshold is not None: logger.info(f"Setting confidence_threshold to " f"{best_confidence_threshold} based on results") - # params.postprocessing.confidence_threshold = best_confidence_threshold else: raise ValueError(f"Cannot compute metrics: Invalid confidence threshold!") - # self._task_environment.set_configurable_parameters(params) logger.info(f"F-measure after evaluation: {f_measure_metrics.f_measure.value}") output_result_set.performance = f_measure_metrics.get_performance() @@ -311,7 +344,6 @@ def unload(self): logger.warning(f"Done unloading. " f"Torch is still occupying {torch.cuda.memory_allocated()} bytes of GPU memory") - def export(self, export_type: ExportType, output_model: ModelEntity): @@ -347,8 +379,7 @@ def export(self, def save_model(self, output_model: ModelEntity): buffer = io.BytesIO() - hyperparams = self._task_environment.get_hyper_parameters(OTEDetectionConfig) - hyperparams_str = ids_to_strings(cfg_helper.convert(hyperparams, dict, enum_to_str=True)) + hyperparams_str = ids_to_strings(cfg_helper.convert(self._hyperparams, dict, enum_to_str=True)) labels = {label.name: label.color.rgb_tuple for label in self._labels} modelinfo = {'model': self._model.state_dict(), 'config': hyperparams_str, 'labels': labels, 'VERSION': 1} torch.save(modelinfo, buffer) diff --git a/mmdet/apis/ote/apis/detection/ote_utils.py b/mmdet/apis/ote/apis/detection/ote_utils.py index e66fdfb0e8a..563fd3f3491 100644 --- a/mmdet/apis/ote/apis/detection/ote_utils.py +++ b/mmdet/apis/ote/apis/detection/ote_utils.py @@ -15,8 +15,12 @@ import colorsys import importlib import random +from typing import Dict, Any import numpy as np +from ote_sdk.entities.dataset_item import DatasetItemEntity +from ote_sdk.entities.datasets import DatasetEntity +from ote_sdk.entities.image import Image import yaml from ote_sdk.entities.color import Color from ote_sdk.entities.label import LabelEntity @@ -115,3 +119,35 @@ def __init__(self, num_test_steps, update_progress_callback: UpdateProgressCallb def on_test_batch_end(self, batch=None, logs=None): super().on_test_batch_end(batch, logs) self.update_progress_callback(self.get_progress()) + + +def dump_dataset_item(item: DatasetItemEntity): + dump = { + 'subset': item.subset, + 'numpy': item.numpy, + 'roi': item.roi, + 'annotation_scene': item.annotation_scene + } + return dump + + +def load_dataset_item(dump: Dict[str, Any]): + return DatasetItemEntity( + media=Image(dump['numpy']), + annotation_scene=dump['annotation_scene'], + roi=dump['roi'], + subset=dump['subset']) + + +def dump_dataset(dataset: DatasetEntity): + dump = { + 'purpose': dataset.purpose, + 'items': list(dump_dataset_item(item) for item in dataset) + } + return dump + + +def load_dataset(dump: Dict[str, Any]): + return DatasetEntity( + items=[load_dataset_item(i) for i in dump['items']], + purpose=dump['purpose']) From 25d8d909b7bcbcc4b51bd67c4b9d297694ad598c Mon Sep 17 00:00:00 2001 From: "Druzhkov, Pavel" Date: Thu, 7 Oct 2021 18:49:30 +0300 Subject: [PATCH 2/9] up --- mmdet/apis/ote/apis/detection/config_utils.py | 5 +- .../apis/ote/apis/detection/inference_task.py | 55 +++++++++++++++---- 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/mmdet/apis/ote/apis/detection/config_utils.py b/mmdet/apis/ote/apis/detection/config_utils.py index e086c1fcbe5..7907d3f502c 100644 --- a/mmdet/apis/ote/apis/detection/config_utils.py +++ b/mmdet/apis/ote/apis/detection/config_utils.py @@ -158,9 +158,10 @@ def config_from_string(config_string: str) -> Config: return Config.fromfile(temp_file.name) -def save_config_to_file(config: Config): +def save_config_to_file(config: Config, filepath: Optional[str] = None): """ Dump the full config to a file. Filename is 'config.py', it is saved in the current work_dir. """ - filepath = os.path.join(config.work_dir, 'config.py') + if filepath is None: + filepath = os.path.join(config.work_dir, 'config.py') config_string = config_to_string(config) with open(filepath, 'w') as f: f.write(config_string) diff --git a/mmdet/apis/ote/apis/detection/inference_task.py b/mmdet/apis/ote/apis/detection/inference_task.py index ad49390ab48..495659030b6 100644 --- a/mmdet/apis/ote/apis/detection/inference_task.py +++ b/mmdet/apis/ote/apis/detection/inference_task.py @@ -45,7 +45,7 @@ from ote_sdk.usecases.tasks.interfaces.unload_interface import IUnload from mmdet.apis import export_model, single_gpu_test -from mmdet.apis.ote.apis.detection.config_utils import patch_config, prepare_for_testing, set_hyperparams +from mmdet.apis.ote.apis.detection.config_utils import patch_config, prepare_for_testing, save_config_to_file, set_hyperparams from mmdet.apis.ote.apis.detection.configuration import OTEDetectionConfig from mmdet.apis.ote.apis.detection.ote_utils import InferenceProgressCallback from mmdet.datasets import build_dataloader, build_dataset @@ -148,18 +148,51 @@ def _create_model(config: Config, from_scratch: bool = False): model = build_detector(model_cfg) return model - def _dump_model(self): - hyperparams_str = ids_to_strings(cfg_helper.convert(self._hyperparams, dict, enum_to_str=True)) - labels = {label.name: label.color.rgb_tuple for label in self._labels} - weights = self._model.state_dict() - config = self._config + + def __getstate__(self): + from ote_sdk.configuration.helper import convert + + model = { + 'weights': self._model.state_dict(), + 'config': self._config, + } + environment = { + 'model_template': self._task_environment.model_template, + 'hyperparams': convert(self._hyperparams, str), + 'label_schema': self._task_environment.label_schema, + } return { - 'hyperparams': hyperparams_str, - 'labels': labels, - 'weights': weights, - 'config': config + 'environment': environment, + 'model': model, } + + def __setstate__(self, state): + from ote_sdk.configuration.helper import create + from dataclasses import asdict + import yaml + + with tempfile.TemporaryDirectory() as tmpdir: + model_template = state['environment']['model_template'] + save_config_to_file(state['model']['config'], os.path.join(tmpdir, 'model.py')) + with open(os.path.join(tmpdir, 'template.yaml'), 'wt') as f: + yaml.dump(asdict(model_template), f) + model_template.model_template_path = os.path.join(tmpdir, 'template.yaml') + + hyperparams = create(state['environment']['hyperparams']) + label_schema = state['environment']['label_schema'] + environment = TaskEnvironment( + model_template=model_template, + model=None, + hyper_parameters=hyperparams, + label_schema=label_schema, + ) + self.__init__(environment) + + self._model.load_state_dict(state['model']['weights']) + self._config = state['model']['config'] + + def infer(self, dataset: DatasetEntity, inference_parameters: Optional[InferenceParameters] = None) -> DatasetEntity: """ Analyzes a dataset using the latest inference model. """ @@ -171,7 +204,7 @@ def infer(self, dataset: DatasetEntity, inference_parameters: Optional[Inference dump_dict = { 'class_name': class_name, 'entrypoint': func_name, - 'model': self._dump_model(), + 'task': self, 'arguments': { 'dataset': dump_dataset(dataset), # 'inference_parameters': inference_parameters From df34e50c66d27f67de295bf7805d16ae3afa7f61 Mon Sep 17 00:00:00 2001 From: "Druzhkov, Pavel" Date: Fri, 8 Oct 2021 11:36:42 +0300 Subject: [PATCH 3/9] decorate --- mmdet/apis/ote/apis/detection/debug.py | 61 +++++++++++++++++++ .../apis/ote/apis/detection/inference_task.py | 23 ++----- mmdet/apis/ote/apis/detection/train_task.py | 2 + 3 files changed, 68 insertions(+), 18 deletions(-) create mode 100644 mmdet/apis/ote/apis/detection/debug.py diff --git a/mmdet/apis/ote/apis/detection/debug.py b/mmdet/apis/ote/apis/detection/debug.py new file mode 100644 index 00000000000..025271980b6 --- /dev/null +++ b/mmdet/apis/ote/apis/detection/debug.py @@ -0,0 +1,61 @@ +import logging +import pickle +from functools import wraps + +from mmdet.apis.ote.apis.detection.ote_utils import dump_dataset + +logger = logging.getLogger(__name__) + + +def debug_trace(func): + @wraps(func) + def wrapped_function(self, *args, **kwargs): + class_name = self.__class__.__name__ + func_name = func.__name__ + if self._hyperparams.debug_parameters.enable_debug_dump: + dump_dict = { + 'class_name': class_name, + 'entrypoint': func_name, + 'task': self, + } + if func_name not in debug_trace_registry: + raise ValueError(f'Debug tracing is not implemented for {func_name} method.') + dump_dict['arguments'] = debug_trace_registry[func_name](self, *args, **kwargs) + logger.warning(f'Saving debug dump for {class_name}.{func_name} call to {self._debug_dump_file_path}') + with open(self._debug_dump_file_path, 'ab') as fp: + pickle.dump(dump_dict, fp) + return func(self, *args, **kwargs) + return wrapped_function + + +def infer_debug_trace(self, dataset, inference_parameters=None): + return {'dataset': dump_dataset(dataset)} + + +def evaluate_debug_trace(self, output_resultset, evaluation_metric=None): + return { + 'output_resultset': { + 'purpose': output_resultset.purpose, + 'ground_truth_dataset' : dump_dataset(output_resultset.ground_truth_dataset), + 'prediction_dataset' : dump_dataset(output_resultset.prediction_dataset) + }, + 'evaluation_metric': evaluation_metric, + } + +def export_debug_trace(self, export_type, output_model): + return { + 'export_type': export_type + } + +def train_debug_trace(self, dataset, output_model, train_parameters=None): + return { + 'dataset': dump_dataset(dataset), + 'train_parameters': None if train_parameters is None else {'resume': train_parameters.resume} + } + +debug_trace_registry = { + 'infer': infer_debug_trace, + 'train': train_debug_trace, + 'evaluate': evaluate_debug_trace, + 'export': export_debug_trace, +} diff --git a/mmdet/apis/ote/apis/detection/inference_task.py b/mmdet/apis/ote/apis/detection/inference_task.py index 495659030b6..5e8a332b945 100644 --- a/mmdet/apis/ote/apis/detection/inference_task.py +++ b/mmdet/apis/ote/apis/detection/inference_task.py @@ -47,6 +47,7 @@ from mmdet.apis import export_model, single_gpu_test from mmdet.apis.ote.apis.detection.config_utils import patch_config, prepare_for_testing, save_config_to_file, set_hyperparams from mmdet.apis.ote.apis.detection.configuration import OTEDetectionConfig +from mmdet.apis.ote.apis.detection.debug import debug_trace from mmdet.apis.ote.apis.detection.ote_utils import InferenceProgressCallback from mmdet.datasets import build_dataloader, build_dataset from mmdet.models import build_detector @@ -193,27 +194,10 @@ def __setstate__(self, state): self._config = state['model']['config'] + @debug_trace def infer(self, dataset: DatasetEntity, inference_parameters: Optional[InferenceParameters] = None) -> DatasetEntity: """ Analyzes a dataset using the latest inference model. """ - if self._hyperparams.debug_parameters.enable_debug_dump: - from mmdet.apis.ote.apis.detection.ote_utils import dump_dataset - - class_name = self.__class__.__name__ - func_name = 'infer' - dump_dict = { - 'class_name': class_name, - 'entrypoint': func_name, - 'task': self, - 'arguments': { - 'dataset': dump_dataset(dataset), - # 'inference_parameters': inference_parameters - } - } - logger.warning(f'Saving debug dump for {class_name}.{func_name} call to {self._debug_dump_file_path}') - with open(self._debug_dump_file_path, 'ab') as fp: - pickle.dump(dump_dict, fp) - set_hyperparams(self._config, self._hyperparams) if inference_parameters is not None: @@ -299,6 +283,7 @@ def _infer_detector(model: torch.nn.Module, config: Config, dataset: DatasetEnti return eval_predictions, metric + @debug_trace def evaluate(self, output_result_set: ResultSetEntity, evaluation_metric: Optional[str] = None): @@ -377,6 +362,8 @@ def unload(self): logger.warning(f"Done unloading. " f"Torch is still occupying {torch.cuda.memory_allocated()} bytes of GPU memory") + + @debug_trace def export(self, export_type: ExportType, output_model: ModelEntity): diff --git a/mmdet/apis/ote/apis/detection/train_task.py b/mmdet/apis/ote/apis/detection/train_task.py index 5d22d1a2e34..67b7a1e6fd0 100644 --- a/mmdet/apis/ote/apis/detection/train_task.py +++ b/mmdet/apis/ote/apis/detection/train_task.py @@ -30,6 +30,7 @@ from mmdet.apis import train_detector from mmdet.apis.ote.apis.detection.config_utils import prepare_for_training, set_hyperparams +from mmdet.apis.ote.apis.detection.debug import debug_trace from mmdet.apis.ote.apis.detection.inference_task import OTEDetectionInferenceTask from mmdet.apis.ote.apis.detection.ote_utils import TrainingProgressCallback from mmdet.apis.ote.extension.utils.hooks import OTELoggerHook @@ -64,6 +65,7 @@ def _generate_training_metrics_group(self, learning_curves) -> Optional[List[Met return output + @debug_trace def train(self, dataset: DatasetEntity, output_model: ModelEntity, train_parameters: Optional[TrainParameters] = None): """ Trains a model on a dataset """ From 52d04f9116fb67d80bc4788aca8899f37be94814 Mon Sep 17 00:00:00 2001 From: "Druzhkov, Pavel" Date: Fri, 8 Oct 2021 11:53:14 +0300 Subject: [PATCH 4/9] clean up --- mmdet/apis/ote/apis/detection/debug.py | 38 +++++++++++++++++++++- mmdet/apis/ote/apis/detection/ote_utils.py | 36 -------------------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/mmdet/apis/ote/apis/detection/debug.py b/mmdet/apis/ote/apis/detection/debug.py index 025271980b6..5b1018c5170 100644 --- a/mmdet/apis/ote/apis/detection/debug.py +++ b/mmdet/apis/ote/apis/detection/debug.py @@ -1,8 +1,11 @@ import logging import pickle from functools import wraps +from typing import Dict, Any -from mmdet.apis.ote.apis.detection.ote_utils import dump_dataset +from ote_sdk.entities.dataset_item import DatasetItemEntity +from ote_sdk.entities.datasets import DatasetEntity +from ote_sdk.entities.image import Image logger = logging.getLogger(__name__) @@ -59,3 +62,36 @@ def train_debug_trace(self, dataset, output_model, train_parameters=None): 'evaluate': evaluate_debug_trace, 'export': export_debug_trace, } + + +def dump_dataset_item(item: DatasetItemEntity): + dump = { + 'subset': item.subset, + 'numpy': item.numpy, + 'roi': item.roi, + 'annotation_scene': item.annotation_scene + } + return dump + + +def load_dataset_item(dump: Dict[str, Any]): + return DatasetItemEntity( + media=Image(dump['numpy']), + annotation_scene=dump['annotation_scene'], + roi=dump['roi'], + subset=dump['subset']) + + +def dump_dataset(dataset: DatasetEntity): + dump = { + 'purpose': dataset.purpose, + 'items': list(dump_dataset_item(item) for item in dataset) + } + return dump + + +def load_dataset(dump: Dict[str, Any]): + return DatasetEntity( + items=[load_dataset_item(i) for i in dump['items']], + purpose=dump['purpose']) + diff --git a/mmdet/apis/ote/apis/detection/ote_utils.py b/mmdet/apis/ote/apis/detection/ote_utils.py index 563fd3f3491..e66fdfb0e8a 100644 --- a/mmdet/apis/ote/apis/detection/ote_utils.py +++ b/mmdet/apis/ote/apis/detection/ote_utils.py @@ -15,12 +15,8 @@ import colorsys import importlib import random -from typing import Dict, Any import numpy as np -from ote_sdk.entities.dataset_item import DatasetItemEntity -from ote_sdk.entities.datasets import DatasetEntity -from ote_sdk.entities.image import Image import yaml from ote_sdk.entities.color import Color from ote_sdk.entities.label import LabelEntity @@ -119,35 +115,3 @@ def __init__(self, num_test_steps, update_progress_callback: UpdateProgressCallb def on_test_batch_end(self, batch=None, logs=None): super().on_test_batch_end(batch, logs) self.update_progress_callback(self.get_progress()) - - -def dump_dataset_item(item: DatasetItemEntity): - dump = { - 'subset': item.subset, - 'numpy': item.numpy, - 'roi': item.roi, - 'annotation_scene': item.annotation_scene - } - return dump - - -def load_dataset_item(dump: Dict[str, Any]): - return DatasetItemEntity( - media=Image(dump['numpy']), - annotation_scene=dump['annotation_scene'], - roi=dump['roi'], - subset=dump['subset']) - - -def dump_dataset(dataset: DatasetEntity): - dump = { - 'purpose': dataset.purpose, - 'items': list(dump_dataset_item(item) for item in dataset) - } - return dump - - -def load_dataset(dump: Dict[str, Any]): - return DatasetEntity( - items=[load_dataset_item(i) for i in dump['items']], - purpose=dump['purpose']) From ed9dcd069dab60785b87895926ddc4241f5319cf Mon Sep 17 00:00:00 2001 From: Ilya Krylov Date: Fri, 22 Oct 2021 09:34:13 +0300 Subject: [PATCH 5/9] dump to dedicated folder --- mmdet/apis/ote/apis/detection/debug.py | 2 ++ mmdet/apis/ote/apis/detection/inference_task.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/mmdet/apis/ote/apis/detection/debug.py b/mmdet/apis/ote/apis/detection/debug.py index 5b1018c5170..5d2510a15ce 100644 --- a/mmdet/apis/ote/apis/detection/debug.py +++ b/mmdet/apis/ote/apis/detection/debug.py @@ -1,4 +1,5 @@ import logging +import os import pickle from functools import wraps from typing import Dict, Any @@ -25,6 +26,7 @@ def wrapped_function(self, *args, **kwargs): raise ValueError(f'Debug tracing is not implemented for {func_name} method.') dump_dict['arguments'] = debug_trace_registry[func_name](self, *args, **kwargs) logger.warning(f'Saving debug dump for {class_name}.{func_name} call to {self._debug_dump_file_path}') + os.makedirs(os.path.dirname(self._debug_dump_file_path), exist_ok=True) with open(self._debug_dump_file_path, 'ab') as fp: pickle.dump(dump_dict, fp) return func(self, *args, **kwargs) diff --git a/mmdet/apis/ote/apis/detection/inference_task.py b/mmdet/apis/ote/apis/detection/inference_task.py index edec2e7bb7b..db2d64ef694 100644 --- a/mmdet/apis/ote/apis/detection/inference_task.py +++ b/mmdet/apis/ote/apis/detection/inference_task.py @@ -13,11 +13,13 @@ # and limitations under the License. import copy +import datetime import io import logging import os import pickle import shutil +import socket import tempfile import warnings from typing import List, Optional, Tuple @@ -56,10 +58,19 @@ logger.setLevel(logging.INFO) +def get_dump_file_path(): + full_path = os.path.join( + '/NOUS' if os.path.exists('/NOUS') else '/tmp', + 'debug_dumps', + socket.gethostname(), + datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + '.pkl') + return full_path + + class OTEDetectionInferenceTask(IInferenceTask, IExportTask, IEvaluationTask, IUnload): _task_environment: TaskEnvironment - _debug_dump_file_path: str = '/tmp/debug_dump.pkl' + _debug_dump_file_path: str = get_dump_file_path() def __init__(self, task_environment: TaskEnvironment): """" From 2548a1d8caf8de3c07dd43b87ebd3d6f28825cbe Mon Sep 17 00:00:00 2001 From: Ilya Krylov Date: Mon, 25 Oct 2021 16:00:50 +0300 Subject: [PATCH 6/9] add script for debugging --- mmdet/apis/ote/apis/detection/debug.py | 77 ++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 4 deletions(-) diff --git a/mmdet/apis/ote/apis/detection/debug.py b/mmdet/apis/ote/apis/detection/debug.py index 5d2510a15ce..ee1bf9b7016 100644 --- a/mmdet/apis/ote/apis/detection/debug.py +++ b/mmdet/apis/ote/apis/detection/debug.py @@ -37,27 +37,30 @@ def infer_debug_trace(self, dataset, inference_parameters=None): return {'dataset': dump_dataset(dataset)} -def evaluate_debug_trace(self, output_resultset, evaluation_metric=None): +def evaluate_debug_trace(self, output_result_set, evaluation_metric=None): return { 'output_resultset': { - 'purpose': output_resultset.purpose, - 'ground_truth_dataset' : dump_dataset(output_resultset.ground_truth_dataset), - 'prediction_dataset' : dump_dataset(output_resultset.prediction_dataset) + 'purpose': output_result_set.purpose, + 'ground_truth_dataset' : dump_dataset(output_result_set.ground_truth_dataset), + 'prediction_dataset' : dump_dataset(output_result_set.prediction_dataset) }, 'evaluation_metric': evaluation_metric, } + def export_debug_trace(self, export_type, output_model): return { 'export_type': export_type } + def train_debug_trace(self, dataset, output_model, train_parameters=None): return { 'dataset': dump_dataset(dataset), 'train_parameters': None if train_parameters is None else {'resume': train_parameters.resume} } + debug_trace_registry = { 'infer': infer_debug_trace, 'train': train_debug_trace, @@ -97,3 +100,69 @@ def load_dataset(dump: Dict[str, Any]): items=[load_dataset_item(i) for i in dump['items']], purpose=dump['purpose']) + +if __name__ == '__main__': + import argparse + from ote_sdk.entities.model import ModelEntity, ModelStatus + from ote_sdk.entities.resultset import ResultSetEntity + + + def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('dump_path') + return parser.parse_args() + + + def main(): + args = parse_args() + assert os.path.exists(args.dump_path) + + output_model = None + train_dataset = None + + with open(args.dump_path, 'rb') as f: + while True: + try: + dump = pickle.load(f) + except EOFError: + break + + task = dump['task'] + method_args = {} + + entrypoint = dump['entrypoint'] + print(f'{type(task)=}, {entrypoint=}') + + if entrypoint == 'train': + method_args['dataset'] = load_dataset(dump['arguments']['dataset']) + train_dataset = method_args['dataset'] + method_args['output_model'] = ModelEntity( + method_args['dataset'], + task._task_environment, + model_status=ModelStatus.NOT_READY) + output_model = method_args['output_model'] + method_args['train_parameters'] = None + elif entrypoint == 'infer': + method_args['dataset'] = load_dataset(dump['arguments']['dataset']) + method_args['inference_parameters'] = None + elif entrypoint == 'export': + method_args['output_model'] = ModelEntity( + train_dataset, + task._task_environment, + model_status=ModelStatus.NOT_READY) + output_model = method_args['output_model'] + method_args['export_type'] = dump['arguments']['export_type'] + elif entrypoint == 'evaluate': + method_args['output_result_set'] = ResultSetEntity( + model=output_model, + ground_truth_dataset=load_dataset(dump['arguments']['output_resultset']['ground_truth_dataset']), + prediction_dataset=load_dataset(dump['arguments']['output_resultset']['prediction_dataset']) + ) + method_args['evaluation_metric'] = dump['arguments']['evaluation_metric'] + else: + raise RuntimeError(f'Unknown {entrypoint=}') + + output = getattr(task, entrypoint)(**method_args) + print(f'\nOutput {type(output)=}\n\n\n\n') + + main() From b0946fa3bf358b3bd171261fdd67c4bf035e4423 Mon Sep 17 00:00:00 2001 From: "Druzhkov, Pavel" Date: Thu, 18 Nov 2021 13:25:47 +0300 Subject: [PATCH 7/9] add user interaction to the debug script --- mmdet/apis/ote/apis/detection/debug.py | 35 +++++++++++++++++++ .../apis/ote/apis/detection/inference_task.py | 12 ++++--- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/mmdet/apis/ote/apis/detection/debug.py b/mmdet/apis/ote/apis/detection/debug.py index a7e66ea45df..93719d7e263 100644 --- a/mmdet/apis/ote/apis/detection/debug.py +++ b/mmdet/apis/ote/apis/detection/debug.py @@ -1,4 +1,5 @@ import datetime +import logging import os import pickle import socket @@ -135,16 +136,45 @@ def main(): with open(args.dump_path, 'rb') as f: while True: + print('reading dump record...') + logger = get_root_logger() + logger.setLevel(logging.ERROR) try: dump = pickle.load(f) except EOFError: + print('no more records found in the dump file') break + logger.setLevel(logging.INFO) task = dump['task'] + # Disable debug dump when replay another debug dump + task._task_environment.get_hyper_parameters().debug_parameters.enable_debug_dump = False method_args = {} entrypoint = dump['entrypoint'] + print('*' * 80) + print(f'{type(task)=}, {entrypoint=}') + print('=' * 80) + + while True: + action = input('[r]eplay, [s]kip or [q]uit : [r] ') + action = action.lower() + if action == '': + action = 'r' + if action not in {'r', 's', 'q'}: + continue + else: + break + + if action == 's': + print('skipping the step replay') + continue + if action == 'q': + print('quiting dump replay session') + exit(0) + + print('replaying the step') if entrypoint == 'train': method_args['dataset'] = load_dataset(dump['arguments']['dataset']) @@ -166,6 +196,11 @@ def main(): output_model = method_args['output_model'] method_args['export_type'] = dump['arguments']['export_type'] elif entrypoint == 'evaluate': + output_model = ModelEntity( + DatasetEntity(), + task._task_environment, + model_status=ModelStatus.SUCCESS) + output_model.configuration.label_schema = task._task_environment.label_schema method_args['output_result_set'] = ResultSetEntity( model=output_model, ground_truth_dataset=load_dataset(dump['arguments']['output_resultset']['ground_truth_dataset']), diff --git a/mmdet/apis/ote/apis/detection/inference_task.py b/mmdet/apis/ote/apis/detection/inference_task.py index 3fa760e3cc9..ffce9cc62e9 100644 --- a/mmdet/apis/ote/apis/detection/inference_task.py +++ b/mmdet/apis/ote/apis/detection/inference_task.py @@ -17,9 +17,9 @@ import logging import os import shutil +import subprocess import tempfile import warnings -from subprocess import run from typing import List, Optional, Tuple import numpy as np @@ -65,11 +65,11 @@ def __init__(self, task_environment: TaskEnvironment): Task for inference object detection models using OTEDetection. """ - print('ENVIRONMENT:') + logger.info('ENVIRONMENT:') for name, val in collect_env().items(): - print(f'{name}: {val}') - print('pip list:') - run('pip list', shell=True, check=True) + logger.info(f'{name}: {val}') + logger.info('pip list:') + logger.info(subprocess.check_output(['pip', 'list'], universal_newlines=True)) self._task_environment = task_environment @@ -168,6 +168,7 @@ def __getstate__(self): model = { 'weights': self._model.state_dict(), 'config': self._config, + 'confidence_threshold': self.confidence_threshold, } environment = { 'model_template': self._task_environment.model_template, @@ -204,6 +205,7 @@ def __setstate__(self, state): self._model.load_state_dict(state['model']['weights']) self._config = state['model']['config'] + self.confidence_threshold = state['model']['confidence_threshold'] def _add_predictions_to_dataset(self, prediction_results, dataset, confidence_threshold=0.0): From 7d68759fdf0d1f9150aad6ce631e5ca7e818973f Mon Sep 17 00:00:00 2001 From: "Druzhkov, Pavel" Date: Thu, 18 Nov 2021 13:28:56 +0300 Subject: [PATCH 8/9] disable debug dump for sample script --- mmdet/apis/ote/sample/sample.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mmdet/apis/ote/sample/sample.py b/mmdet/apis/ote/sample/sample.py index 0491fcf5130..e36586bd3ef 100644 --- a/mmdet/apis/ote/sample/sample.py +++ b/mmdet/apis/ote/sample/sample.py @@ -118,6 +118,7 @@ def main(args): params.learning_parameters.num_iters = 5 params.learning_parameters.learning_rate_warmup_iters = 1 params.learning_parameters.batch_size = 2 + params.debug_parameters.enable_debug_dump = False logger.info('Setup environment') environment = TaskEnvironment(model=None, hyper_parameters=params, label_schema=labels_schema, model_template=model_template) From b09a16510639a526fa2cdc4f9708ea304bffa496 Mon Sep 17 00:00:00 2001 From: "Druzhkov, Pavel" Date: Thu, 18 Nov 2021 13:47:46 +0300 Subject: [PATCH 9/9] disable debug dumps in tests --- tests/test_ote_api.py | 1 + tests/test_ote_training.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/test_ote_api.py b/tests/test_ote_api.py index 9f06f1f152e..dc65c55bd53 100644 --- a/tests/test_ote_api.py +++ b/tests/test_ote_api.py @@ -163,6 +163,7 @@ def setup_configurable_parameters(self, template_dir, num_iters=10): hyper_parameters.learning_parameters.num_checkpoints = 1 hyper_parameters.postprocessing.result_based_confidence_threshold = False hyper_parameters.postprocessing.confidence_threshold = 0.1 + hyper_parameters.debug_parameters.enable_debug_dump = False return hyper_parameters, model_template @e2e_pytest_api diff --git a/tests/test_ote_training.py b/tests/test_ote_training.py index e9f4831843b..10716f03bf3 100644 --- a/tests/test_ote_training.py +++ b/tests/test_ote_training.py @@ -243,6 +243,7 @@ def _run_ote_training(self, data_collector): logger.debug('Set hyperparameters') params = create(self.model_template.hyper_parameters.data) + params.debug_parameters.enable_debug_dump = False params.learning_parameters.num_iters = self.num_training_iters if self.num_training_iters < 20: num_checkpoints = 2