From b440625567f7572f53ebd904f6c427715935f83a Mon Sep 17 00:00:00 2001 From: OJ Date: Fri, 8 Apr 2022 11:29:49 +0100 Subject: [PATCH 1/2] Fixed issue with loading checkpoint during hyper parameter tuning --- keras_declarative/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keras_declarative/util.py b/keras_declarative/util.py index 601e3f4..2cac10d 100644 --- a/keras_declarative/util.py +++ b/keras_declarative/util.py @@ -312,7 +312,7 @@ def _build_hypermodel(self, hp): model.load_weights( self._get_checkpoint_fname( - history_trial.trial_id, history_trial.best_step + history_trial.trial_id #, history_trial.best_step ) ) return model From 7d884692dcc8c56210f68cc9622cc9575d5df543 Mon Sep 17 00:00:00 2001 From: OJ Date: Tue, 25 Oct 2022 16:10:09 +0100 Subject: [PATCH 2/2] setting Memory Growth at start of training --- keras_declarative/workflows.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/keras_declarative/workflows.py b/keras_declarative/workflows.py index 71f42b2..fc16cfa 100644 --- a/keras_declarative/workflows.py +++ b/keras_declarative/workflows.py @@ -26,6 +26,12 @@ import numpy as np import pandas as pd import tensorflow as tf +try: + physical_devices = tf.config.experimental.list_physical_devices('GPU') + tf.config.experimental.set_memory_growth(physical_devices[0], True) + print('setting Memory Growth (for simultaneous preproc and training OJ)') +except: + print('FAILED: setting Memory Growth (for simultaneous preproc and training OJ)') import tensorflow_datasets as tfds import tensorflow_io as tfio