Merge branch 'develop'

Author: urbanmatthias

.gitignore

@@ -30,12 +30,6 @@ jobs.txt
 *autonet.egg-info
 *.simg
 
-# Metalearning data
-/metalearning_data/
-/metalearning_comparison_results/
-/meta_outputs/
-/metamodels/
-
 
 # Datasets
 /datasets/

README.md

@@ -22,10 +22,11 @@ $ git checkout develop
 
 Install pytorch: 
 https://pytorch.org/
- 
-Install autonet
+
+Install Auto-PyTorch:
 
 ```sh
+$ cat requirements.txt | xargs -n 1 -L 1 pip install
 $ python setup.py install
 ```
 
@@ -46,7 +47,12 @@ X_train, X_test, y_train, y_test = \
         sklearn.model_selection.train_test_split(X, y, random_state=1)
 
 # running Auto-PyTorch
-autoPyTorch = AutoNetClassification(log_level='info', max_runtime=300, min_budget=30, max_budget=90)
+autoPyTorch = AutoNetClassification("tiny_cs",  # config preset
+                                    log_level='info',
+                                    max_runtime=300,
+                                    min_budget=30,
+                                    max_budget=90)
+
 autoPyTorch.fit(X_train, y_train, validation_split=0.3)
 y_pred = autoPyTorch.predict(X_test)
 
@@ -57,6 +63,67 @@ More examples with datasets:
 
 ```sh
 $ cd examples/
+
+```
+
+## Configuration
+
+How to configure Auto-PyTorch for your needs:
+
+```py
+
+# Print all possible configuration options.
+AutoNetClassification().print_help()
+
+# You can use the constructor to configure Auto-PyTorch.
+autoPyTorch = AutoNetClassification(log_level='info', max_runtime=300, min_budget=30, max_budget=90)
+
+# You can overwrite this configuration in each fit call.
+autoPyTorch.fit(X_train, y_train, log_level='debug', max_runtime=900, min_budget=50, max_budget=150)
+
+# You can use presets to configure the config space.
+# Available presets: full_cs, medium_cs (default), tiny_cs.
+# These are defined in autoPyTorch/core/presets.
+# tiny_cs is recommended if you want fast results with few resources.
+# full_cs is recommended if you have many resources and a very high search budget.
+autoPyTorch = AutoNetClassification("full_cs")
+
+# Enable or disable components using the Auto-PyTorch config:
+autoPyTorch = AutoNetClassification(networks=["resnet", "shapedresnet", "mlpnet", "shapedmlpnet"])
+
+# You can take a look at the search space.
+# Each hyperparameter belongs to a node in Auto-PyTorch's ML Pipeline.
+# The names of the hyperparameters are prefixed with the name of the node: NodeName:hyperparameter_name.
+# If a hyperparameter belongs to a component: NodeName:component_name:hyperparameter_name.
+autoPyTorch.get_hyperparameter_search_space()
+
+# You can configure the search space of every hyperparameter of every component:
+from autoPyTorch import HyperparameterSearchSpaceUpdates
+search_space_updates = HyperparameterSearchSpaceUpdates()
+
+search_space_updates.append(node_name="NetworkSelector",
+                            hyperparameter="shapedresnet:activation",
+                            value_range=["relu", "sigmoid"])
+search_space_updates.append(node_name="NetworkSelector",
+                            hyperparameter="shapedresnet:blocks_per_group",
+                            value_range=[2,5],
+                            log=False)
+autoPyTorch = AutoNetClassification(hyperparameter_search_space_updates=search_space_updates)
+```
+
+Enable ensemble building:
+
+```py
+from autoPyTorch import AutoNetEnsemble
+autoPyTorchEnsemble = AutoNetEnsemble(AutoNetClassification, "tiny_cs", max_runtime=300, min_budget=30, max_budget=90)
+
+```
+
+Disable pynisher if you experience issues when using cuda:
+
+```py
+autoPyTorch = AutoNetClassification("tiny_cs", log_level='info', max_runtime=300, min_budget=30, max_budget=90, cuda=True, use_pynisher=False)
+
 ```
 
 ## License

autoPyTorch/components/ensembles/__init__.py

@@ -27,7 +27,6 @@ def fit(self, base_models_predictions, true_targets, model_identifiers):
         self
 
         """
-        pass
 
     @abstractmethod
     def predict(self, base_models_predictions):
@@ -42,7 +41,6 @@ def predict(self, base_models_predictions):
         -------
         array : [n_data_points]
         """
-        self
 
     @abstractmethod
     def get_models_with_weights(self, models):

autoPyTorch/components/ensembles/abstract_ensemble.py

@@ -4,14 +4,14 @@
 This file contains the different learning rate schedulers of AutoNet.
 """
 
+from autoPyTorch.utils.config_space_hyperparameter import add_hyperparameter, get_hyperparameter
+
 import torch
 import torch.optim.lr_scheduler as lr_scheduler
 
 import ConfigSpace as CS
 import ConfigSpace.hyperparameters as CSH
 
-from autoPyTorch.components.lr_scheduler.lr_schedulers_config import CSConfig
-
 __author__ = "Max Dippel, Michael Burkart and Matthias Urban"
 __version__ = "0.0.1"
 __license__ = "BSD"
@@ -29,8 +29,7 @@ def _get_scheduler(self, optimizer, config):
         raise ValueError('Override the method _get_scheduler and do not call the base class implementation')
 
     @staticmethod
-    def get_config_space(*args, **kwargs):
-        # currently no use but might come in handy in the future
+    def get_config_space():
         return CS.ConfigurationSpace()
 
 class SchedulerNone(AutoNetLearningRateSchedulerBase):
@@ -44,12 +43,13 @@ def _get_scheduler(self, optimizer, config):
         return lr_scheduler.StepLR(optimizer=optimizer, step_size=config['step_size'], gamma=config['gamma'], last_epoch=-1)
 
     @staticmethod
-    def get_config_space(*args, **kwargs):
+    def get_config_space(
+        step_size=(1, 10),
+        gamma=(0.001, 0.9)
+    ):
         cs = CS.ConfigurationSpace()
-        config = CSConfig['step_lr']
-        cs.add_hyperparameter(CSH.UniformIntegerHyperparameter('step_size', lower=config['step_size'][0], upper=config['step_size'][1]))
-        cs.add_hyperparameter(CSH.UniformFloatHyperparameter('gamma', lower=config['gamma'][0], upper=config['gamma'][1]))
-        cs.add_configuration_space(prefix='', delimiter='', configuration_space=AutoNetLearningRateSchedulerBase.get_config_space(*args, **kwargs))
+        add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'step_size', step_size)
+        add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'gamma', gamma)
         return cs
 
 class SchedulerExponentialLR(AutoNetLearningRateSchedulerBase):
@@ -58,11 +58,11 @@ def _get_scheduler(self, optimizer, config):
         return lr_scheduler.ExponentialLR(optimizer=optimizer, gamma=config['gamma'], last_epoch=-1)
 
     @staticmethod
-    def get_config_space(*args, **kwargs):
+    def get_config_space(
+        gamma=(0.8, 0.9999)
+    ):
         cs = CS.ConfigurationSpace()
-        config = CSConfig['exponential_lr']
-        cs.add_hyperparameter(CSH.UniformFloatHyperparameter('gamma', lower=config['gamma'][0], upper=config['gamma'][1]))
-        cs.add_configuration_space(prefix='', delimiter='', configuration_space=AutoNetLearningRateSchedulerBase.get_config_space(*args, **kwargs))
+        add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'gamma', gamma)
         return cs
 
 class SchedulerReduceLROnPlateau(AutoNetLearningRateSchedulerBase):
@@ -71,12 +71,13 @@ def _get_scheduler(self, optimizer, config):
         return lr_scheduler.ReduceLROnPlateau(optimizer=optimizer)
 
     @staticmethod
-    def get_config_space(*args, **kwargs):
+    def get_config_space(
+        factor=(0.05, 0.5),
+        patience=(3, 10)
+    ):
         cs = CS.ConfigurationSpace()
-        config = CSConfig['reduce_on_plateau']
-        cs.add_hyperparameter(CSH.UniformFloatHyperparameter('factor', lower=config['factor'][0], upper=config['factor'][1]))
-        cs.add_hyperparameter(CSH.UniformIntegerHyperparameter('patience', lower=config['patience'][0], upper=config['patience'][1]))
-        cs.add_configuration_space(prefix='', delimiter='', configuration_space=AutoNetLearningRateSchedulerBase.get_config_space(*args, **kwargs))
+        add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'factor', factor)
+        add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'patience', patience)
         return cs
 
 class SchedulerCyclicLR(AutoNetLearningRateSchedulerBase):
@@ -96,13 +97,15 @@ def l(epoch):
         return lr_scheduler.LambdaLR(optimizer=optimizer, lr_lambda=l, last_epoch=-1)
 
     @staticmethod
-    def get_config_space(*args, **kwargs):
+    def get_config_space(
+        max_factor=(1.0, 2),
+        min_factor=(0.001, 1.0),
+        cycle_length=(3, 10)
+    ):
         cs = CS.ConfigurationSpace()
-        config = CSConfig['cyclic_lr']
-        cs.add_hyperparameter(CSH.UniformFloatHyperparameter('max_factor', lower=config['max_factor'][0], upper=config['max_factor'][1]))
-        cs.add_hyperparameter(CSH.UniformFloatHyperparameter('min_factor', lower=config['min_factor'][0], upper=config['min_factor'][1]))
-        cs.add_hyperparameter(CSH.UniformIntegerHyperparameter('cycle_length', lower=config['cycle_length'][0], upper=config['cycle_length'][1]))
-        cs.add_configuration_space(prefix='', delimiter='', configuration_space=AutoNetLearningRateSchedulerBase.get_config_space(*args, **kwargs))
+        add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'max_factor', max_factor)
+        add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'min_factor', min_factor)
+        add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'cycle_length', cycle_length)
         return cs
 
 class SchedulerCosineAnnealingWithRestartsLR(AutoNetLearningRateSchedulerBase):
@@ -114,12 +117,13 @@ def _get_scheduler(self, optimizer, config):
         return scheduler
 
     @staticmethod
-    def get_config_space(*args, **kwargs):
+    def get_config_space(
+        T_max=(1, 20),
+        T_mult=(1.0, 2.0)
+    ):
         cs = CS.ConfigurationSpace()
-        config = CSConfig['cosine_annealing_lr']
-        cs.add_hyperparameter(CSH.UniformIntegerHyperparameter('T_max', lower=config['T_max'][0], upper=config['T_max'][1]))
-        cs.add_hyperparameter(CSH.UniformFloatHyperparameter('T_mult', lower=config['T_mult'][0], upper=config['T_mult'][1]))
-        cs.add_configuration_space(prefix='', delimiter='', configuration_space=AutoNetLearningRateSchedulerBase.get_config_space(*args, **kwargs))
+        add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'T_max', T_max)
+        add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'T_mult', T_mult)
         return cs
 
 

autoPyTorch/components/lr_scheduler/lr_schedulers.py

@@ -5,7 +5,7 @@
 
 
 def balanced_accuracy(y_pred, y_true):
-    return _balanced_accuracy(np.argmax(y_pred, axis=1), np.argmax(y_true, axis=1)) * 100
+    return _balanced_accuracy(np.argmax(y_true, axis=1), np.argmax(y_pred, axis=1)) * 100
 
 
 def _balanced_accuracy(solution, prediction):

autoPyTorch/components/metrics/balanced_accuracy.py

@@ -11,6 +11,7 @@
 import torch.nn as nn
 import numpy as np
 
+from autoPyTorch.utils.config_space_hyperparameter import get_hyperparameter, add_hyperparameter
 from autoPyTorch.components.preprocessing.preprocessor_base import PreprocessorBase
 
 __author__ = "Max Dippel, Michael Burkart and Matthias Urban"
@@ -76,16 +77,23 @@ def _create_ee_layers(self, in_features):
          return layers
 
     @staticmethod
-    def get_config_space(categorical_features=None):
+    def get_config_space(
+        categorical_features=None,
+        min_unique_values_for_embedding=((3, 300), True),
+        dimension_reduction=(0, 1),
+        **kwargs
+    ):
         # dimension of entity embedding layer is a hyperparameter
         if categorical_features is None or not any(categorical_features):
             return CS.ConfigurationSpace()
         cs = CS.ConfigurationSpace()
-        min_hp = CSH.UniformIntegerHyperparameter("min_unique_values_for_embedding", lower=3, upper=300, default_value=3, log=True)
+        min_hp = get_hyperparameter(CSH.UniformIntegerHyperparameter, "min_unique_values_for_embedding", min_unique_values_for_embedding)
         cs.add_hyperparameter(min_hp)
         for i in range(len([x for x in categorical_features if x])):
-            ee_dimensions = CSH.UniformFloatHyperparameter("dimension_reduction_" + str(i), lower=0, upper=1, default_value=1, log=False)
-            cs.add_hyperparameter(ee_dimensions)
+            ee_dimensions_hp = get_hyperparameter(CSH.UniformFloatHyperparameter, "dimension_reduction_" + str(i),
+                kwargs.pop("dimension_reduction_" + str(i), dimension_reduction))
+            cs.add_hyperparameter(ee_dimensions_hp)
+        assert len(kwargs) == 0, "Invalid hyperparameter updates for learned embedding: %s" % str(kwargs)
         return cs
 
 class NoEmbedding(nn.Module):

autoPyTorch/components/networks/feature/embedding.py

@@ -11,6 +11,7 @@
 import torch.nn as nn
 
 from autoPyTorch.components.networks.base_net import BaseFeatureNet
+from autoPyTorch.utils.config_space_hyperparameter import add_hyperparameter, get_hyperparameter
 
 __author__ = "Max Dippel, Michael Burkart and Matthias Urban"
 __version__ = "0.0.1"
@@ -46,35 +47,38 @@ def _add_layer(self, layers, in_features, out_features, layer_id):
             layers.append(nn.Dropout(self.config["dropout_%d" % layer_id]))
 
     @staticmethod
-    def get_config_space(user_updates=None):
+    def get_config_space(
+        num_layers=((1, 15), False),
+        num_units=((10, 1024), True),
+        activation=('sigmoid', 'tanh', 'relu'),
+        dropout=(0.0, 0.8),
+        use_dropout=(True, False),
+        **kwargs
+    ):
         cs = CS.ConfigurationSpace()
-        range_num_layers=(1, 15)
-        range_num_units=(10, 1024)
-        possible_activations=('sigmoid', 'tanh', 'relu')
-        range_dropout=(0.0, 0.8)
-        
-        if user_updates is not None and 'num_layers' in user_updates:
-            range_num_layers = user_updates['num_layers']
-
-        num_layers = CSH.UniformIntegerHyperparameter('num_layers', lower=range_num_layers[0], upper=range_num_layers[1])
-        cs.add_hyperparameter(num_layers)
-        use_dropout = cs.add_hyperparameter(CS.CategoricalHyperparameter("use_dropout", [True, False], default_value=True))
-
-        for i in range(1, range_num_layers[1] + 1):
-            n_units = CSH.UniformIntegerHyperparameter("num_units_%d" % i,
-                lower=range_num_units[0], upper=range_num_units[1], log=True)
-            cs.add_hyperparameter(n_units)
-            dropout = CSH.UniformFloatHyperparameter("dropout_%d" % i, lower=range_dropout[0], upper=range_dropout[1])
-            cs.add_hyperparameter(dropout)
-            dropout_condition_1 = CS.EqualsCondition(dropout, use_dropout, True)
-
-            if i > range_num_layers[0]:
-                cs.add_condition(CS.GreaterThanCondition(n_units, num_layers, i - 1))
-
-                dropout_condition_2 = CS.GreaterThanCondition(dropout, num_layers, i - 1)
-                cs.add_condition(CS.AndConjunction(dropout_condition_1, dropout_condition_2))
-            else:
-                cs.add_condition(dropout_condition_1)
+
+        num_layers_hp = get_hyperparameter(CSH.UniformIntegerHyperparameter, 'num_layers', num_layers)
+        cs.add_hyperparameter(num_layers_hp)
+        use_dropout_hp = add_hyperparameter(cs, CS.CategoricalHyperparameter, "use_dropout", use_dropout)
+
+        for i in range(1, num_layers[0][1] + 1):
+            n_units_hp = get_hyperparameter(CSH.UniformIntegerHyperparameter, "num_units_%d" % i, kwargs.pop("num_units_%d" % i, num_units))
+            cs.add_hyperparameter(n_units_hp)
+
+            if i > num_layers[0][0]:
+                cs.add_condition(CS.GreaterThanCondition(n_units_hp, num_layers_hp, i - 1))
+
+            if True in use_dropout:
+                dropout_hp = get_hyperparameter(CSH.UniformFloatHyperparameter, "dropout_%d" % i, kwargs.pop("dropout_%d" % i, dropout))
+                cs.add_hyperparameter(dropout_hp)
+                dropout_condition_1 = CS.EqualsCondition(dropout_hp, use_dropout_hp, True)
+
+                if i > num_layers[0][0]:
+                    dropout_condition_2 = CS.GreaterThanCondition(dropout_hp, num_layers_hp, i - 1)
+                    cs.add_condition(CS.AndConjunction(dropout_condition_1, dropout_condition_2))
+                else:
+                    cs.add_condition(dropout_condition_1)
 
-        cs.add_hyperparameter(CSH.CategoricalHyperparameter('activation', possible_activations))
+        add_hyperparameter(cs, CSH.CategoricalHyperparameter,'activation', activation)
+        assert len(kwargs) == 0, "Invalid hyperparameter updates for mlpnet: %s" % str(kwargs)
         return(cs)