From cbbd1b8f3a84780bdcd2af4eb45eb02d9fc0f2ff Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 27 Apr 2021 15:25:12 +0200 Subject: [PATCH 01/70] Textfile um den branch zu testen --- Test_Sonja.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Test_Sonja.txt diff --git a/Test_Sonja.txt b/Test_Sonja.txt new file mode 100644 index 0000000..09c235e --- /dev/null +++ b/Test_Sonja.txt @@ -0,0 +1,2 @@ +Hallo Sonja! +OMG das bin ja ich! From 64814ab2d5cb752744dd18679713c0bc05cd72d3 Mon Sep 17 00:00:00 2001 From: Sonja Date: Mon, 10 May 2021 14:44:11 +0200 Subject: [PATCH 02/70] Added new class for Hyperparameter Optimization --- sparseSpACE/HP_Optimization.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 sparseSpACE/HP_Optimization.py diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py new file mode 100644 index 0000000..af9bd7f --- /dev/null +++ b/sparseSpACE/HP_Optimization.py @@ -0,0 +1,30 @@ +import sparseSpACE.DEMachineLearning as deml + +print("I am working!") + +#Basically copied code from Tutorial_DEMachineLearning +def perform_optimization_mock(): + samples = 500 + dimension = 2 + labels = 6 + sklearn_dataset = deml.datasets.make_moons(n_samples=samples, noise=0.15) + data = deml.DataSet(sklearn_dataset, name='Input_Set') + classification = deml.Classification(data, split_percentage=0.8, split_evenly=True, shuffle_data=True) + classification.perform_classification(masslumping=True, lambd=0.0, minimum_level=1, maximum_level=5, print_metrics=True) + evaluation = classification.evaluate() + print("Percentage of correct mappings",evaluation["Percentage correct"]) + # also more testing data can be added and the results can be printed immediately + + + data_copy = data.copy() # deepcopied + data_copy.scale_range((0.005, 0.995)) # scaled + part0, part1 = data_copy.split_pieces(0.5) # split + data_copy = part0.concatenate(part1) # concatenated + data_copy.set_name('2nd_Set') # renamed + data_copy.remove_labels(0.5) # freed of some label assignments to samples + without_labels, with_labels = data_copy.split_without_labels() + with_labels.set_name("Test_new_data") + classification.test_data(with_labels, print_output=True) + +perform_optimization_mock() +print("I was working!") From 44e00d41e93e16c01175c77343bb7cf929720f29 Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 11 May 2021 09:46:24 +0200 Subject: [PATCH 03/70] Added rudimentary comparison of performance to mock function --- sparseSpACE/HP_Optimization.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index af9bd7f..2d8a7b0 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -4,18 +4,32 @@ #Basically copied code from Tutorial_DEMachineLearning def perform_optimization_mock(): + #prepare Dataset samples = 500 dimension = 2 labels = 6 sklearn_dataset = deml.datasets.make_moons(n_samples=samples, noise=0.15) data = deml.DataSet(sklearn_dataset, name='Input_Set') + #for storing the current best evaluation + best_evaluation = None + #best_evaluation = 0.7 + print("original best evaluation: " + str(best_evaluation)) + #todo: split Dataset classification = deml.Classification(data, split_percentage=0.8, split_evenly=True, shuffle_data=True) - classification.perform_classification(masslumping=True, lambd=0.0, minimum_level=1, maximum_level=5, print_metrics=True) + classification.perform_classification(masslumping=True, lambd=0.0, minimum_level=1, maximum_level=5, print_metrics=True) #True evaluation = classification.evaluate() print("Percentage of correct mappings",evaluation["Percentage correct"]) - # also more testing data can be added and the results can be printed immediately + #currently only looking at Percentage correct for evaluating + current_evaluation = evaluation["Percentage correct"] + if best_evaluation == None or current_evaluation < best_evaluation: + best_evaluation = current_evaluation + print("Best evaluation is now " + str(best_evaluation)) + else: print("Best evaluation is still " + str(best_evaluation)) + + print("In the end, best evaluation is " + str(best_evaluation)) + """# also more testing data can be added and the results can be printed immediately data_copy = data.copy() # deepcopied data_copy.scale_range((0.005, 0.995)) # scaled part0, part1 = data_copy.split_pieces(0.5) # split @@ -24,7 +38,7 @@ def perform_optimization_mock(): data_copy.remove_labels(0.5) # freed of some label assignments to samples without_labels, with_labels = data_copy.split_without_labels() with_labels.set_name("Test_new_data") - classification.test_data(with_labels, print_output=True) + classification.test_data(with_labels, print_output=True)""" perform_optimization_mock() print("I was working!") From 5a746402474d1713875afce001eca6f7cabaa65f Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 11 May 2021 12:31:00 +0200 Subject: [PATCH 04/70] Turned optimization mock into simple (non-exhaustive) grid search --- sparseSpACE/HP_Optimization.py | 44 ++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 2d8a7b0..0b939cb 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -3,7 +3,7 @@ print("I am working!") #Basically copied code from Tutorial_DEMachineLearning -def perform_optimization_mock(): +def perform_optimization_grid_simple(): #prepare Dataset samples = 500 dimension = 2 @@ -12,21 +12,39 @@ def perform_optimization_mock(): data = deml.DataSet(sklearn_dataset, name='Input_Set') #for storing the current best evaluation best_evaluation = None + best_time = None + #storing the parameters of the best evaluation + best_lambd = None + best_masslump = None #best_evaluation = 0.7 print("original best evaluation: " + str(best_evaluation)) #todo: split Dataset classification = deml.Classification(data, split_percentage=0.8, split_evenly=True, shuffle_data=True) - classification.perform_classification(masslumping=True, lambd=0.0, minimum_level=1, maximum_level=5, print_metrics=True) #True - evaluation = classification.evaluate() - print("Percentage of correct mappings",evaluation["Percentage correct"]) - #currently only looking at Percentage correct for evaluating - current_evaluation = evaluation["Percentage correct"] - if best_evaluation == None or current_evaluation < best_evaluation: - best_evaluation = current_evaluation - print("Best evaluation is now " + str(best_evaluation)) - else: print("Best evaluation is still " + str(best_evaluation)) - - print("In the end, best evaluation is " + str(best_evaluation)) + classification.perform_classification(masslumping=True, lambd=0.0, minimum_level=1, maximum_level=5, print_metrics=True) + #perform grid search for a few values of masslumping and lambd + for i in range (0, 11, 10): + for j in [True, False]: + #original settings: classification.perform_classification(masslumping=True, lambd=0.0, minimum_level=1, maximum_level=5, print_metrics=True) + classification = deml.Classification(data, split_percentage=0.8, split_evenly=True, shuffle_data=True) + current_lambd = i/10 + current_masslump = j + print("masslumping = " + str(current_masslump) + "; lambd = " + str(current_lambd)) + classification.perform_classification(masslumping=current_masslump, lambd=current_lambd, minimum_level=1, maximum_level=5, print_metrics=False) + current_time = classification._time_used + print ("current time needed = " + str(current_time)) + evaluation = classification.evaluate() + print("Percentage of correct mappings",evaluation["Percentage correct"]) + #currently only looking at Percentage correct and time needed for evaluating + current_evaluation = evaluation["Percentage correct"] + if best_evaluation == None or current_evaluation > best_evaluation or (current_evaluation == best_evaluation and (best_time == None or best_time>current_time)): + best_evaluation = current_evaluation + best_lambd = current_lambd + best_time = current_time + best_masslump = current_masslump + print("Best evaluation is now " + str(best_evaluation) + " with masslump = " + str (best_masslump) + ", lambd = " + str(best_lambd) + " and time = " + str(best_time)) + else: print("Best evaluation is still " + str(best_evaluation) + " with masslump = " + str (best_masslump) + " and lambd = " + str(best_lambd)+ " and time = " + str(best_time)) + print ("END OF CURRENT EVALUATION \n") + print("In the end, best evaluation is " + str(best_evaluation) + " with masslump = " + str (best_masslump) + " and lambd = " + str(best_lambd)+ " and time = " + str(best_time)) """# also more testing data can be added and the results can be printed immediately @@ -40,5 +58,5 @@ def perform_optimization_mock(): with_labels.set_name("Test_new_data") classification.test_data(with_labels, print_output=True)""" -perform_optimization_mock() +perform_optimization_grid_simple() print("I was working!") From 31038db03dd8914fa4bdf7c9bc93a7b0b5e87a31 Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 11 May 2021 15:34:23 +0200 Subject: [PATCH 05/70] added seed to dataset generation and set shuffle_data=false so function is deterministic --- sparseSpACE/HP_Optimization.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 0b939cb..af5c5c4 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -1,34 +1,35 @@ import sparseSpACE.DEMachineLearning as deml -print("I am working!") +#print("I am working!") #Basically copied code from Tutorial_DEMachineLearning +#prepare Dataset +samples = 500 +dimension = 2 +labels = 6 +sklearn_dataset = deml.datasets.make_moons(n_samples=samples, noise=0.15, random_state=1) +data = deml.DataSet(sklearn_dataset, name='Input_Set') def perform_optimization_grid_simple(): - #prepare Dataset - samples = 500 - dimension = 2 - labels = 6 - sklearn_dataset = deml.datasets.make_moons(n_samples=samples, noise=0.15) - data = deml.DataSet(sklearn_dataset, name='Input_Set') - #for storing the current best evaluation + #sklearn_dataset = deml.datasets.make_moons(n_samples=samples, noise=0.15) + #data = deml.DataSet(sklearn_dataset, name='Input_Set') + #for storing the current best evaluation and time best_evaluation = None best_time = None #storing the parameters of the best evaluation best_lambd = None best_masslump = None - #best_evaluation = 0.7 - print("original best evaluation: " + str(best_evaluation)) + #For Testing best evaluation: best_evaluation = 0.7 + #print("original best evaluation: " + str(best_evaluation)) #todo: split Dataset - classification = deml.Classification(data, split_percentage=0.8, split_evenly=True, shuffle_data=True) - classification.perform_classification(masslumping=True, lambd=0.0, minimum_level=1, maximum_level=5, print_metrics=True) + #original classification: classification = deml.Classification(data, split_percentage=0.8, split_evenly=True, shuffle_data=True) + #original settings: classification.perform_classification(masslumping=True, lambd=0.0, minimum_level=1, maximum_level=5, print_metrics=True) #perform grid search for a few values of masslumping and lambd for i in range (0, 11, 10): for j in [True, False]: - #original settings: classification.perform_classification(masslumping=True, lambd=0.0, minimum_level=1, maximum_level=5, print_metrics=True) - classification = deml.Classification(data, split_percentage=0.8, split_evenly=True, shuffle_data=True) + classification = deml.Classification(data, split_percentage=0.8, split_evenly=True, shuffle_data=False) current_lambd = i/10 current_masslump = j - print("masslumping = " + str(current_masslump) + "; lambd = " + str(current_lambd)) + print("current masslumping = " + str(current_masslump) + "; current lambd = " + str(current_lambd)) classification.perform_classification(masslumping=current_masslump, lambd=current_lambd, minimum_level=1, maximum_level=5, print_metrics=False) current_time = classification._time_used print ("current time needed = " + str(current_time)) @@ -59,4 +60,4 @@ def perform_optimization_grid_simple(): classification.test_data(with_labels, print_output=True)""" perform_optimization_grid_simple() -print("I was working!") +#print("I was working!") From 276c1838854e64e718dd98302a99e1a92b6ee365 Mon Sep 17 00:00:00 2001 From: Sonja Date: Mon, 31 May 2021 09:36:21 +0200 Subject: [PATCH 06/70] Added function to evaluate specific combinations of HP --- sparseSpACE/HP_Optimization.py | 73 +++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index af5c5c4..6cdbcd4 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -7,8 +7,12 @@ samples = 500 dimension = 2 labels = 6 +#vllt anderes dataset ausprobieren?, vllt höhere dimensionen, bis zu dim=10. Note: moons immer 2d, aber z.B. +#gaussian quantiles, classification.. kann man einstellen sklearn_dataset = deml.datasets.make_moons(n_samples=samples, noise=0.15, random_state=1) data = deml.DataSet(sklearn_dataset, name='Input_Set') +data_moons = data.copy() +data_moons.set_name('Moon_Set') def perform_optimization_grid_simple(): #sklearn_dataset = deml.datasets.make_moons(n_samples=samples, noise=0.15) #data = deml.DataSet(sklearn_dataset, name='Input_Set') @@ -18,46 +22,49 @@ def perform_optimization_grid_simple(): #storing the parameters of the best evaluation best_lambd = None best_masslump = None + best_min_lv = None + best_one_vs_others = None #For Testing best evaluation: best_evaluation = 0.7 #print("original best evaluation: " + str(best_evaluation)) #todo: split Dataset #original classification: classification = deml.Classification(data, split_percentage=0.8, split_evenly=True, shuffle_data=True) #original settings: classification.perform_classification(masslumping=True, lambd=0.0, minimum_level=1, maximum_level=5, print_metrics=True) #perform grid search for a few values of masslumping and lambd - for i in range (0, 11, 10): - for j in [True, False]: - classification = deml.Classification(data, split_percentage=0.8, split_evenly=True, shuffle_data=False) - current_lambd = i/10 - current_masslump = j - print("current masslumping = " + str(current_masslump) + "; current lambd = " + str(current_lambd)) - classification.perform_classification(masslumping=current_masslump, lambd=current_lambd, minimum_level=1, maximum_level=5, print_metrics=False) - current_time = classification._time_used - print ("current time needed = " + str(current_time)) - evaluation = classification.evaluate() - print("Percentage of correct mappings",evaluation["Percentage correct"]) - #currently only looking at Percentage correct and time needed for evaluating - current_evaluation = evaluation["Percentage correct"] - if best_evaluation == None or current_evaluation > best_evaluation or (current_evaluation == best_evaluation and (best_time == None or best_time>current_time)): - best_evaluation = current_evaluation - best_lambd = current_lambd - best_time = current_time - best_masslump = current_masslump - print("Best evaluation is now " + str(best_evaluation) + " with masslump = " + str (best_masslump) + ", lambd = " + str(best_lambd) + " and time = " + str(best_time)) - else: print("Best evaluation is still " + str(best_evaluation) + " with masslump = " + str (best_masslump) + " and lambd = " + str(best_lambd)+ " and time = " + str(best_time)) - print ("END OF CURRENT EVALUATION \n") - print("In the end, best evaluation is " + str(best_evaluation) + " with masslump = " + str (best_masslump) + " and lambd = " + str(best_lambd)+ " and time = " + str(best_time)) + lambd_levels = 10 + #Note: vllt besser logarithmisch, sehr kleine Werte oft besser (1/10, 1/100, 1/1000) + for cur_lambd in range (0, lambd_levels+1, 5): + #Note: lamd vor Allem sinnvoll wenn masslumping aus ist, sollte kaum unterschied machen wenn + for cur_massl in [True, False]: + for cur_min_lv in range (1, 4, 1): + for cur_one_vs_others in [True, False]: + classification = deml.Classification(data, split_percentage=0.8, split_evenly=True, shuffle_data=False) + print("current masslumping = " + str(cur_massl) + "; current lambd = " + str(cur_lambd/lambd_levels) + "; current min lv = " + str(cur_min_lv) + "; current one vs others = " + str(cur_one_vs_others)) + classification.perform_classification(masslumping=cur_massl, lambd=cur_lambd/lambd_levels, minimum_level=cur_min_lv, maximum_level=5, one_vs_others=cur_one_vs_others, print_metrics=False) + cur_time = classification._time_used + print ("current time needed = " + str(cur_time)) + evaluation = classification.evaluate() + print("Percentage of correct mappings",evaluation["Percentage correct"]) + #currently only looking at Percentage correct and time needed for evaluating + cur_evaluation = evaluation["Percentage correct"] + if best_evaluation == None or cur_evaluation > best_evaluation or (cur_evaluation == best_evaluation and (best_time == None or best_time>cur_time)): + best_evaluation = cur_evaluation + best_lambd = cur_lambd + best_time = cur_time + best_masslump = cur_massl + best_min_lv = cur_min_lv + best_one_vs_others = cur_one_vs_others + print("Best evaluation is now " + str(best_evaluation) + " with masslump = " + str(best_masslump) + ", lambd = " + str(best_lambd) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others) + " and time = " + str(best_time)) + else: print("Best evaluation is still " + str(best_evaluation) + " with masslump = " + str(best_masslump) + ", lambd = " + str(best_lambd) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others) + " and time = " + str(best_time)) + print ("END OF CURRENT EVALUATION \n") + print("In the end, best evaluation is " + str(best_evaluation) + " with masslump = " + str (best_masslump) + ", lamb = " + str(best_lambd) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others) + " and time = " + str(best_time)) +def perform_optimization_at(cur_data, cur_lambd: float, cur_massl: bool, cur_min_lv: int, cur_one_vs_others: bool): + classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) + classification.perform_classification(masslumping=cur_massl, lambd=cur_lambd, minimum_level=cur_min_lv, maximum_level=5, one_vs_others=cur_one_vs_others, print_metrics=False) + evaluation = classification.evaluate() + print("Percentage of correct mappings",evaluation["Percentage correct"]) - """# also more testing data can be added and the results can be printed immediately - data_copy = data.copy() # deepcopied - data_copy.scale_range((0.005, 0.995)) # scaled - part0, part1 = data_copy.split_pieces(0.5) # split - data_copy = part0.concatenate(part1) # concatenated - data_copy.set_name('2nd_Set') # renamed - data_copy.remove_labels(0.5) # freed of some label assignments to samples - without_labels, with_labels = data_copy.split_without_labels() - with_labels.set_name("Test_new_data") - classification.test_data(with_labels, print_output=True)""" -perform_optimization_grid_simple() +perform_optimization_at(data_moons, 0.0, False, 1, False) +#perform_optimization_grid_simple() #print("I was working!") From 33a7e8a0b4c9a6f4aaf7d04559c48a4372b357a2 Mon Sep 17 00:00:00 2001 From: Sonja Date: Mon, 7 Jun 2021 19:27:42 +0200 Subject: [PATCH 07/70] created BO method with mock values --- sparseSpACE/HP_Optimization.py | 60 ++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 6cdbcd4..dbcc81e 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -1,19 +1,10 @@ import sparseSpACE.DEMachineLearning as deml +import numpy +import scipy +import math -#print("I am working!") - -#Basically copied code from Tutorial_DEMachineLearning -#prepare Dataset -samples = 500 -dimension = 2 -labels = 6 -#vllt anderes dataset ausprobieren?, vllt höhere dimensionen, bis zu dim=10. Note: moons immer 2d, aber z.B. -#gaussian quantiles, classification.. kann man einstellen -sklearn_dataset = deml.datasets.make_moons(n_samples=samples, noise=0.15, random_state=1) -data = deml.DataSet(sklearn_dataset, name='Input_Set') -data_moons = data.copy() -data_moons.set_name('Moon_Set') -def perform_optimization_grid_simple(): +#performs Grid Optimization +def perform_GO_classification(data): #sklearn_dataset = deml.datasets.make_moons(n_samples=samples, noise=0.15) #data = deml.DataSet(sklearn_dataset, name='Input_Set') #for storing the current best evaluation and time @@ -58,13 +49,48 @@ def perform_optimization_grid_simple(): print ("END OF CURRENT EVALUATION \n") print("In the end, best evaluation is " + str(best_evaluation) + " with masslump = " + str (best_masslump) + ", lamb = " + str(best_lambd) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others) + " and time = " + str(best_time)) -def perform_optimization_at(cur_data, cur_lambd: float, cur_massl: bool, cur_min_lv: int, cur_one_vs_others: bool): +#returns evaluation for certain Parameters on a certain data set +def perform_evaluation_at(cur_data, cur_lambd: float, cur_massl: bool, cur_min_lv: int, cur_one_vs_others: bool): classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) classification.perform_classification(masslumping=cur_massl, lambd=cur_lambd, minimum_level=cur_min_lv, maximum_level=5, one_vs_others=cur_one_vs_others, print_metrics=False) evaluation = classification.evaluate() print("Percentage of correct mappings",evaluation["Percentage correct"]) + return evaluation["Percentage correct"] + +#TODO: will perform Bayesian Optimization; Inputs are the data, amount of iterations(amt_it) +def perform_BO_classification(data, amt_it: int): + print("I should do Bayesian Optimization for " + str(amt_it) + " iteration!") + #dummy value for beta + beta = 0.5 + #use squared exp. kernel as covariance function, with parameters sigma_k and l_k + sigma_k = 1 + l_k = 0.5 + k = lambda x: x + mu = lambda x: x**2+4 + sigma = lambda x: x**2 + alpha = lambda x: mu(x)+(beta**(0.5))*sigma(x) + for i in range (0, amt_it, 1): + beta = get_beta(i) + print("beta: " + str(beta) + " mu(x): " + str(mu(2)) + " sigma(x): " + str(sigma(2))) + print(alpha(2)) +#TODO: will calculate and return current beta +def get_beta(cur_it: int): + beta = cur_it/2 + return beta -perform_optimization_at(data_moons, 0.0, False, 1, False) -#perform_optimization_grid_simple() +#print("I am working!") + +#Basically copied code from Tutorial_DEMachineLearning +#prepare Dataset +samples = 500 +dimension = 2 +labels = 6 +#vllt anderes dataset ausprobieren?, vllt höhere dimensionen, bis zu dim=10. Note: moons immer 2d, aber z.B. +#gaussian quantiles, classification.. kann man einstellen +sklearn_dataset = deml.datasets.make_moons(n_samples=samples, noise=0.15, random_state=1) +data1 = deml.DataSet(sklearn_dataset, name='Input_Set') +data_moons = data1.copy() +data_moons.set_name('Moon_Set') +perform_BO_classification(data_moons, 5) #print("I was working!") From 69ce4a6f4f37fa3e548f12c93ae75832202fea4f Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 8 Jun 2021 14:32:54 +0200 Subject: [PATCH 08/70] started method to initialize evidence set --- sparseSpACE/HP_Optimization.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index dbcc81e..912e9a1 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -1,6 +1,6 @@ import sparseSpACE.DEMachineLearning as deml -import numpy -import scipy +import numpy as np +import scipy as sp import math #performs Grid Optimization @@ -58,17 +58,20 @@ def perform_evaluation_at(cur_data, cur_lambd: float, cur_massl: bool, cur_min_l return evaluation["Percentage correct"] #TODO: will perform Bayesian Optimization; Inputs are the data, amount of iterations(amt_it) -def perform_BO_classification(data, amt_it: int): +def perform_BO_classification(data, amt_it: int, dim: int): + print("I should do Bayesian Optimization for " + str(amt_it) + " iteration!") #dummy value for beta beta = 0.5 - #use squared exp. kernel as covariance function, with parameters sigma_k and l_k + #use squared exp. kernel as covariance function k, with parameters sigma_k and l_k sigma_k = 1 l_k = 0.5 - k = lambda x: x + #if creation of covariance matrices is done in another function k(x) needs to be outside + k = lambda x, y: (sigma_k**2)*math.exp((-0.5/l_k**2)*np.linalg.norm(x-y)**2) mu = lambda x: x**2+4 - sigma = lambda x: x**2 - alpha = lambda x: mu(x)+(beta**(0.5))*sigma(x) + sigma_sqrd = lambda x: k(x, x) + sigma = lambda x: math.sqrt(sigma_sqrd(x)) + alpha = lambda x: mu(x)+(math.sqrt(beta))*sigma(x) for i in range (0, amt_it, 1): beta = get_beta(i) print("beta: " + str(beta) + " mu(x): " + str(mu(2)) + " sigma(x): " + str(sigma(2))) @@ -79,6 +82,16 @@ def get_beta(cur_it: int): beta = cur_it/2 return beta +#TODO: will return initial evidence set +def get_init_evidence(data, dim: int, amt_it: int): + #hard-coded x-values. Order:[lambd: float, massl:bool, min_lv: 0 Date: Tue, 15 Jun 2021 10:56:52 +0200 Subject: [PATCH 09/70] Added usage of evidence set. creation of K_matrix and k_vector and used them in the respective functions. Struggling to maximize alpha --- sparseSpACE/HP_Optimization.py | 103 ++++++++++++++++++++++---- sparseSpACE/HP_Optimization.py.save | 107 ++++++++++++++++++++++++++++ 2 files changed, 195 insertions(+), 15 deletions(-) create mode 100644 sparseSpACE/HP_Optimization.py.save diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 912e9a1..b5cb32f 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -2,6 +2,8 @@ import numpy as np import scipy as sp import math +import random +from scipy.optimize import fmin #performs Grid Optimization def perform_GO_classification(data): @@ -59,30 +61,103 @@ def perform_evaluation_at(cur_data, cur_lambd: float, cur_massl: bool, cur_min_l #TODO: will perform Bayesian Optimization; Inputs are the data, amount of iterations(amt_it) def perform_BO_classification(data, amt_it: int, dim: int): - - print("I should do Bayesian Optimization for " + str(amt_it) + " iteration!") + #notes how many x values currently are in the evidence set - starts with amt_HP+1 + amt_HP = 4 + cur_amt_x: int = amt_HP+1 + print("I should do Bayesian Optimization for " + str(amt_it) + " iterations!") + C = create_evidence_set(data, amt_it, amt_HP) + C_x=C[0] + C_y=C[1] + print(C) + K_matr = get_cov_matrix(C_x, cur_amt_x) + print(K_matr) #dummy value for beta beta = 0.5 - #use squared exp. kernel as covariance function k, with parameters sigma_k and l_k - sigma_k = 1 - l_k = 0.5 - #if creation of covariance matrices is done in another function k(x) needs to be outside - k = lambda x, y: (sigma_k**2)*math.exp((-0.5/l_k**2)*np.linalg.norm(x-y)**2) - mu = lambda x: x**2+4 - sigma_sqrd = lambda x: k(x, x) + mu = lambda x: get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(C_y)) + sigma_sqrd = lambda x: k(x, x)-get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(get_cov_vector(C_x, x, cur_amt_x))) sigma = lambda x: math.sqrt(sigma_sqrd(x)) alpha = lambda x: mu(x)+(math.sqrt(beta))*sigma(x) + #negates alpha bc maximum has to be found + alpha_neg = lambda x: -alpha(x) + stupid_value=np.array([1, 1, 1, 1]) for i in range (0, amt_it, 1): + print("iteration: " + str(i)) beta = get_beta(i) - print("beta: " + str(beta) + " mu(x): " + str(mu(2)) + " sigma(x): " + str(sigma(2))) - print(alpha(2)) + print("beta: " + str(beta) + " mu(x): " + str(mu(stupid_value)) + " sigma(x): " + str(sigma(stupid_value))) + print(alpha(stupid_value)) + #value that will be evaluated and added to the evidence set + print(fmin(alpha_neg, [0, 0, 0, 0])) + #new_x=fmin(alpha_neg, [0, 0, 0, 0]) + #print("current new_x: " + str(new_x) + " with function value: " + str(alpha(new_x))) + cur_amt_x += 1 + + +#use squared exp. kernel as covariance function k, with parameters sigma_k and l_k +sigma_k = 1 +l_k = 0.5 +#since creation of covariance matrices is done in another function k(x) needs to be outside +k = lambda x, y: (sigma_k**2)*math.exp((-0.5/l_k**2)*np.linalg.norm(x-y)**2) + +#cur_length is needed so that rest of matrix stays Id +def get_cov_matrix(x, cur_length: int): + K = np.identity(len(x)) + for i in range (0, cur_length, 1): + for j in range (0, cur_length, 1): + K[i][j]=k(x[i], x[j]) + return K + +#returns the vector k for a certain input new_x +def get_cov_vector(ev_x, new_x, cur_length: int): + #somehow the rest of the function would still work even if x is wrong size, but no idea why / what it does. Better to check + if(len(ev_x[0]) != len(new_x)): + print("Input x has the wrong size") + return None + k_vec = np.zeros(len(ev_x)) + for i in range (0, cur_length, 1): + k_vec[i] = k(ev_x[i], new_x) + return k_vec + +def create_evidence_set(data, amt_it: int, dim_HP: int, is_random: bool = True): + x = np.zeros(shape=(amt_it+dim_HP+1, dim_HP)) + y = np.zeros((amt_it+dim_HP+1)) + print(x) + if(is_random): + for i in range(0, dim_HP+1, 1): + new_x = create_random_x() + while(check_if_in_array(new_x, x)): + new_x = create_random_x() + x[i] = new_x + #evaluating takes quite some time, especially for min_lv>1 + y[i] = 0.97 #hard-coded y for faster testing of the rest +#perform_evaluation_at(data, new_x[0], new_x[1], new_x[2], new_x[3]) + #One could hard-code non-random values for testing + return x, y + +#returns a random x for the given purpose +def create_random_x(purpose = "classification"): + if(purpose == "classification"): + return [random.random(), random.randint(0, 1), random.randint(1, 3), random.randint(0, 1)] + else: + print("Invalid Input") + +#checks if array has the element x, returns True if yes. Caution: If x has same length but different dimension to the elements of array this may give unintended results +#maybe later a threshhold can be added if only part of the matrix should be checked +def check_if_in_array(x, array): + if(len(x) != len(array[0])): + #print("not same size") + return False + for i in range(0, len(array), 1): + if((x == array[i]).all()): + return True + break + return False #TODO: will calculate and return current beta def get_beta(cur_it: int): beta = cur_it/2 return beta -#TODO: will return initial evidence set +#Created newer, better method above, but this may be helpful for the hardcoding stuff def get_init_evidence(data, dim: int, amt_it: int): #hard-coded x-values. Order:[lambd: float, massl:bool, min_lv: 0 best_evaluation or (cur_evaluation == best_evaluation and (best_time == None or best_time>cur_time)): + best_evaluation = cur_evaluation + best_lambd = cur_lambd + best_time = cur_time + best_masslump = cur_massl + best_min_lv = cur_min_lv + best_one_vs_others = cur_one_vs_others + print("Best evaluation is now " + str(best_evaluation) + " with masslump = " + str(best_masslump) + ", lambd = " + str(best_lambd) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others) + " and time = " + str(best_time)) + else: print("Best evaluation is still " + str(best_evaluation) + " with masslump = " + str(best_masslump) + ", lambd = " + str(best_lambd) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others) + " and time = " + str(best_time)) + print ("END OF CURRENT EVALUATION \n") + print("In the end, best evaluation is " + str(best_evaluation) + " with masslump = " + str (best_masslump) + ", lamb = " + str(best_lambd) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others) + " and time = " + str(best_time)) + +#returns evaluation for certain Parameters on a certain data set +def perform_evaluation_at(cur_data, cur_lambd: float, cur_massl: bool, cur_min_lv: int, cur_one_vs_others: bool): + classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) + classification.perform_classification(masslumping=cur_massl, lambd=cur_lambd, minimum_level=cur_min_lv, maximum_level=5, one_vs_others=cur_one_vs_others, print_metrics=False) + evaluation = classification.evaluate() + print("Percentage of correct mappings",evaluation["Percentage correct"]) + return evaluation["Percentage correct"] + +#TODO: will perform Bayesian Optimization; Inputs are the data, amount of iterations(amt_it) +def perform_BO_classification(data, amt_it: int, dim: int): + + print("I should do Bayesian Optimization for " + str(amt_it) + " iteration!") + #dummy value for beta + beta = 0.5 + #use squared exp. kernel as covariance function k, with parameters sigma_k and l_k + sigma_k = 1 + l_k = 0.5 + #if creation of covariance matrices is done in another function k(x) needs to be outside + k = lambda x, y: (sigma_k**2)*math.exp((-0.5/l_k**2)*np.linalg.norm(x-y)**2) + mu = lambda x: x**2+4 + sigma_sqrd = lambda x: k(x, x) + sigma = lambda x: math.sqrt(sigma_sqrd(x)) + alpha = lambda x: mu(x)+(math.sqrt(beta))*sigma(x) + for i in range (0, amt_it, 1): + beta = get_beta(i) + print("beta: " + str(beta) + " mu(x): " + str(mu(2)) + " sigma(x): " + str(sigma(2))) + print(alpha(2)) + +#TODO: will calculate and return current beta +def get_beta(cur_it: int): + beta = cur_it/2 + return beta + +#TODO: will return initial evidence set +def get_init_evidence(data, dim: int, amt_it: int): + #hard-coded x-values. Order:[lambd: float, massl:bool, min_lv: 0 Date: Tue, 15 Jun 2021 13:14:06 +0200 Subject: [PATCH 10/70] Implemented BO with naive rounding and arbitrary beta, but haven't revised yet --- sparseSpACE/HP_Optimization.py | 59 +++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index b5cb32f..42e44e8 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -59,39 +59,59 @@ def perform_evaluation_at(cur_data, cur_lambd: float, cur_massl: bool, cur_min_l print("Percentage of correct mappings",evaluation["Percentage correct"]) return evaluation["Percentage correct"] -#TODO: will perform Bayesian Optimization; Inputs are the data, amount of iterations(amt_it) +#TODO: will perform Bayesian Optimization; Currently basically performs BO with Naive rounding and a weird beta +#Note: HP Input of classification is (lambd:float, massl:bool, min_lv:int <4, one_vs_others:bool) def perform_BO_classification(data, amt_it: int, dim: int): #notes how many x values currently are in the evidence set - starts with amt_HP+1 amt_HP = 4 cur_amt_x: int = amt_HP+1 + x_ret = None + y_ret = None print("I should do Bayesian Optimization for " + str(amt_it) + " iterations!") C = create_evidence_set(data, amt_it, amt_HP) C_x=C[0] C_y=C[1] - print(C) + print("Evidence Set: \n" + str(C)) K_matr = get_cov_matrix(C_x, cur_amt_x) print(K_matr) #dummy value for beta beta = 0.5 mu = lambda x: get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(C_y)) sigma_sqrd = lambda x: k(x, x)-get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(get_cov_vector(C_x, x, cur_amt_x))) - sigma = lambda x: math.sqrt(sigma_sqrd(x)) + #takes sqrt of abs(sigma_sqrd) bc otherwise fmin gives an error - might be an overall warning sign tho + sigma = lambda x: math.sqrt(abs(sigma_sqrd(x))) alpha = lambda x: mu(x)+(math.sqrt(beta))*sigma(x) #negates alpha bc maximum has to be found alpha_neg = lambda x: -alpha(x) - stupid_value=np.array([1, 1, 1, 1]) for i in range (0, amt_it, 1): print("iteration: " + str(i)) beta = get_beta(i) - print("beta: " + str(beta) + " mu(x): " + str(mu(stupid_value)) + " sigma(x): " + str(sigma(stupid_value))) - print(alpha(stupid_value)) + print("beta: " + str(beta)) #value that will be evaluated and added to the evidence set - print(fmin(alpha_neg, [0, 0, 0, 0])) - #new_x=fmin(alpha_neg, [0, 0, 0, 0]) - #print("current new_x: " + str(new_x) + " with function value: " + str(alpha(new_x))) - cur_amt_x += 1 - - + new_x=fmin(alpha_neg, [0, 0, 0, 0]) #Note: Vielleicht kann man die Auswertung bounden, damit keine Werte weit weg vom Möglichen rauskommen + print("new x: " + str(new_x) + " with function value: " + str(alpha(new_x))) + #rounds the values of new_x to values usable as HPs - the question is what kind of rounding makes sense, or e.g if lambda should be bounded + lambd = new_x[0] + massl = math.trunc(new_x[1]) #is true if abs val > 1 + min_lv = math.trunc(new_x[2]) + if(min_lv<1): min_lv = 1 + elif (min_lv>3): min_lv = 3 + one_vs_others = math.trunc(new_x[3]) + new_x_rd = [lambd, massl, min_lv, one_vs_others] + print("rounded: " + str(new_x_rd)) + if(not check_if_in_array(new_x_rd, C_x)): + C_x[cur_amt_x] = new_x_rd + C_y[cur_amt_x] = perform_evaluation_at(data, new_x_rd[0], new_x_rd[1], new_x_rd[2], new_x_rd[3]) + cur_amt_x += 1 + else: + print("Dead End! I am crying now") + break + for i in range(0, cur_amt_x, 1): + if(y_ret == None or y_ret1 - y[i] = 0.97 #hard-coded y for faster testing of the rest -#perform_evaluation_at(data, new_x[0], new_x[1], new_x[2], new_x[3]) + y[i] = perform_evaluation_at(data, new_x[0], new_x[1], new_x[2], new_x[3]) #One could hard-code non-random values for testing return x, y @@ -157,17 +175,6 @@ def get_beta(cur_it: int): beta = cur_it/2 return beta -#Created newer, better method above, but this may be helpful for the hardcoding stuff -def get_init_evidence(data, dim: int, amt_it: int): - #hard-coded x-values. Order:[lambd: float, massl:bool, min_lv: 0 Date: Tue, 13 Jul 2021 12:16:11 +0200 Subject: [PATCH 11/70] started method for optimization of beta --- sparseSpACE/HP_Optimization.py | 39 +++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 42e44e8..dbc76a3 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -51,6 +51,18 @@ def perform_GO_classification(data): print ("END OF CURRENT EVALUATION \n") print("In the end, best evaluation is " + str(best_evaluation) + " with masslump = " + str (best_masslump) + ", lamb = " + str(best_lambd) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others) + " and time = " + str(best_time)) +#performs random optimization +def perform_RO_classification(data, amt_it: int, dim: int): + best_evaluation = 0 + best_x = [1, 1, 1, 1] + for i in range (0, amt_it, 1): + x = create_random_x() + new_eval = perform_evaluation_at(data, x[0], x[1], x[2], x[3]) + if new_eval>best_evaluation: + best_x = x + best_evaluation = new_eval + print("Best evaluation in " + str(amt_it) + " random steps: " + str(best_evaluation) + " at " + str(x)) + #returns evaluation for certain Parameters on a certain data set def perform_evaluation_at(cur_data, cur_lambd: float, cur_massl: bool, cur_min_lv: int, cur_one_vs_others: bool): classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) @@ -85,7 +97,7 @@ def perform_BO_classification(data, amt_it: int, dim: int): alpha_neg = lambda x: -alpha(x) for i in range (0, amt_it, 1): print("iteration: " + str(i)) - beta = get_beta(i) + beta = get_beta(i+1) print("beta: " + str(beta)) #value that will be evaluated and added to the evidence set new_x=fmin(alpha_neg, [0, 0, 0, 0]) #Note: Vielleicht kann man die Auswertung bounden, damit keine Werte weit weg vom Möglichen rauskommen @@ -170,11 +182,26 @@ def check_if_in_array(x, array): break return False -#TODO: will calculate and return current beta -def get_beta(cur_it: int): - beta = cur_it/2 +#TODO: will calculate and return current beta. t is current iteration +def get_beta(t: int): + r: float = 1.5 + d: int = 4 + delta: float = 0.1 + a: float = 1 + b: float = 1 + beta = 2*math.log(t**2*2*math.pi**2/3*delta)+2*d*math.log(t**2*d*b*r*math.sqrt(math.log(4*d*a/delta))) + print(beta) return beta +#TODO: get new beta is mu dominated, and new beta and l if not. Currently mock function +def get_new_beta_and_l(cur_beta: float): + global l_k #is l_k should be changed permanently + beta_h = 100 + l_h = 20 + new_beta = (cur_beta+beta_h)/2 + new_l = 10 + return new_beta, new_l + #Basically copied code from Tutorial_DEMachineLearning #prepare Dataset samples = 500 @@ -186,4 +213,6 @@ def get_beta(cur_it: int): data1 = deml.DataSet(sklearn_dataset, name='Input_Set') data_moons = data1.copy() data_moons.set_name('Moon_Set') -perform_BO_classification(data_moons, 6, dimension) +#perform_BO_classification(data_moons, 6, dimension) +#print(get_beta(1)) +perform_RO_classification(data_moons, 10, dimension) From a913c2effabc3f90229ccad2a851766fc2f8c294 Mon Sep 17 00:00:00 2001 From: Sonja Date: Mon, 19 Jul 2021 10:42:36 +0200 Subject: [PATCH 12/70] started acq_x function --- sparseSpACE/HP_Optimization.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index dbc76a3..65aff2e 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -116,6 +116,10 @@ def perform_BO_classification(data, amt_it: int, dim: int): C_y[cur_amt_x] = perform_evaluation_at(data, new_x_rd[0], new_x_rd[1], new_x_rd[2], new_x_rd[3]) cur_amt_x += 1 else: + #erstelle neues beta und l - übergebe cur_length / amt_x damit cov stuff darin berechnet werden kann, C_x damit if in ev set geschaut werden kann + #erstelle neue cov matrix und vector mit neuem l + #berechne neuen wert mit neuem beta und l + #ggf ändere vorher schon cov matrix beim testen auf g? weil gebraucht wird für die berechnung print("Dead End! I am crying now") break for i in range(0, cur_amt_x, 1): @@ -193,15 +197,35 @@ def get_beta(t: int): print(beta) return beta -#TODO: get new beta is mu dominated, and new beta and l if not. Currently mock function -def get_new_beta_and_l(cur_beta: float): - global l_k #is l_k should be changed permanently +#TODO: get new beta and l. Currently mock function +def get_new_beta_and_l(cur_beta: float, cur_length, C_x): + #wsl fnct für get_x mit beta und l sinnvoll - auch für vorher + #l nachher wieder zurücksetzen - bzw lt?? Was ist das, wie berechnet sich das? + global l_k #if l_k should be changed permanently beta_h = 100 l_h = 20 + #0 if in ev set C_x, constant otherwise (3) + p = lambda x: check_if_in_array(x, C_x)*3 + #x[0] is \beta+d\beta, x[1] is l + g = lambda x: x new_beta = (cur_beta+beta_h)/2 new_l = 10 return new_beta, new_l +#TODO acquire new x for l, beta, C_x, cur_amt_x, using GP-UCB +def acq_x(l: float, beta: float, C_x, C_y, cur_amt_x): + print("lol") + #wait no I need to change the l and the cov function D: + K_matr = get_cov_matrix(C_x, cur_amt_x) + print(K_matr) + mu = lambda x: get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(C_y)) + sigma_sqrd = lambda x: k(x, x)-get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(get_cov_vector(C_x, x, cur_amt_x))) + #takes sqrt of abs(sigma_sqrd) bc otherwise fmin gives an error - might be an overall warning sign tho + sigma = lambda x: math.sqrt(abs(sigma_sqrd(x))) + alpha = lambda x: mu(x)+(math.sqrt(beta))*sigma(x) + #negates alpha bc maximum has to be found + alpha_neg = lambda x: -alpha(x) + #Basically copied code from Tutorial_DEMachineLearning #prepare Dataset samples = 500 From afe3f319a0c366921b43d1a51b9c469e410f2e8c Mon Sep 17 00:00:00 2001 From: Sonja Date: Mon, 19 Jul 2021 14:51:43 +0200 Subject: [PATCH 13/70] continued acq_x --- sparseSpACE/HP_Optimization.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 65aff2e..b7ae6eb 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -197,6 +197,10 @@ def get_beta(t: int): print(beta) return beta +#TODO: get l_k and use it +def get_l_k(t: int): + return 0.5 + #TODO: get new beta and l. Currently mock function def get_new_beta_and_l(cur_beta: float, cur_length, C_x): #wsl fnct für get_x mit beta und l sinnvoll - auch für vorher @@ -216,6 +220,9 @@ def get_new_beta_and_l(cur_beta: float, cur_length, C_x): def acq_x(l: float, beta: float, C_x, C_y, cur_amt_x): print("lol") #wait no I need to change the l and the cov function D: + global l_k + old_l=l_k + l_k=l K_matr = get_cov_matrix(C_x, cur_amt_x) print(K_matr) mu = lambda x: get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(C_y)) @@ -225,6 +232,26 @@ def acq_x(l: float, beta: float, C_x, C_y, cur_amt_x): alpha = lambda x: mu(x)+(math.sqrt(beta))*sigma(x) #negates alpha bc maximum has to be found alpha_neg = lambda x: -alpha(x) + new_x=fmin(alpha_neg, [0, 0, 0, 0]) #Note: Vielleicht kann man die Auswertung bounden, damit keine Werte weit weg vom Möglichen rauskommen + print("new x: " + str(new_x) + " with function value: " + str(alpha(new_x))) + return new_x + +def round_x_classification(x): + if len(x) < 4: + print("Input too short! Returning default values") + return [0, 0, 1, 0] + if len(x) > 4: + print("Input too long! Cropping") + #rounds the values of new_x to values usable as HPs - the question is what kind of rounding makes sense + #e.g if lambda should be bounded or more values should be rounded to 0 + lambd = x[0] + massl = math.trunc(x[1]) #is true if abs val > 1 + min_lv = math.trunc(x[2]) + if(min_lv<1): min_lv = 1 + elif (min_lv>3): min_lv = 3 + one_vs_others = math.trunc(x[3]) + new_x_rd = [lambd, massl, min_lv, one_vs_others] + return new_x_rd #Basically copied code from Tutorial_DEMachineLearning #prepare Dataset From e1f719c62096bc632ff296e64efa328e09c0cb05 Mon Sep 17 00:00:00 2001 From: Sonja Date: Fri, 23 Jul 2021 13:50:54 +0200 Subject: [PATCH 14/70] finished (?) get new beta / l method --- sparseSpACE/HP_Optimization.py | 39 +++++++++++----------------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index b7ae6eb..6753cef 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -84,32 +84,15 @@ def perform_BO_classification(data, amt_it: int, dim: int): C_x=C[0] C_y=C[1] print("Evidence Set: \n" + str(C)) - K_matr = get_cov_matrix(C_x, cur_amt_x) - print(K_matr) - #dummy value for beta - beta = 0.5 - mu = lambda x: get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(C_y)) - sigma_sqrd = lambda x: k(x, x)-get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(get_cov_vector(C_x, x, cur_amt_x))) - #takes sqrt of abs(sigma_sqrd) bc otherwise fmin gives an error - might be an overall warning sign tho - sigma = lambda x: math.sqrt(abs(sigma_sqrd(x))) - alpha = lambda x: mu(x)+(math.sqrt(beta))*sigma(x) - #negates alpha bc maximum has to be found - alpha_neg = lambda x: -alpha(x) for i in range (0, amt_it, 1): print("iteration: " + str(i)) beta = get_beta(i+1) - print("beta: " + str(beta)) + l = get_l_k(i) + print("beta_i: " + str(beta)) #value that will be evaluated and added to the evidence set - new_x=fmin(alpha_neg, [0, 0, 0, 0]) #Note: Vielleicht kann man die Auswertung bounden, damit keine Werte weit weg vom Möglichen rauskommen + new_x=acq_x(l, beta, C_x, C_y, cur_amt_x) print("new x: " + str(new_x) + " with function value: " + str(alpha(new_x))) - #rounds the values of new_x to values usable as HPs - the question is what kind of rounding makes sense, or e.g if lambda should be bounded - lambd = new_x[0] - massl = math.trunc(new_x[1]) #is true if abs val > 1 - min_lv = math.trunc(new_x[2]) - if(min_lv<1): min_lv = 1 - elif (min_lv>3): min_lv = 3 - one_vs_others = math.trunc(new_x[3]) - new_x_rd = [lambd, massl, min_lv, one_vs_others] + new_x_rd = round_x_classification(new_x) print("rounded: " + str(new_x_rd)) if(not check_if_in_array(new_x_rd, C_x)): C_x[cur_amt_x] = new_x_rd @@ -202,19 +185,21 @@ def get_l_k(t: int): return 0.5 #TODO: get new beta and l. Currently mock function -def get_new_beta_and_l(cur_beta: float, cur_length, C_x): +def get_new_beta_and_l(cur_beta: float, cur_length, cur_x, C_x, C_y, cur_amt_x): #wsl fnct für get_x mit beta und l sinnvoll - auch für vorher #l nachher wieder zurücksetzen - bzw lt?? Was ist das, wie berechnet sich das? global l_k #if l_k should be changed permanently beta_h = 100 l_h = 20 #0 if in ev set C_x, constant otherwise (3) - p = lambda x: check_if_in_array(x, C_x)*3 + p = lambda x: check_if_in_array(x, C_x)*5 #x[0] is \beta+d\beta, x[1] is l - g = lambda x: x - new_beta = (cur_beta+beta_h)/2 - new_l = 10 - return new_beta, new_l + new_x = lambda x: acq_x(x[1], x[0], C_x,C_y, cur_amt_x) + g = lambda x: (x[0]-cur_beta)+np.linalg.norm(cur_x-new_x(x))+p(new_x(x)) + bounds_g = ((cur_beta, beta_h), (l_k, l_h)) + result = minimize(g, [1, 1], method='L-BFGS-B', bounds=bounds_g) + #result is in the form [new beta, new l] + return result #TODO acquire new x for l, beta, C_x, cur_amt_x, using GP-UCB def acq_x(l: float, beta: float, C_x, C_y, cur_amt_x): From a0451d19803a90c2add669cdd49fade963ae278c Mon Sep 17 00:00:00 2001 From: Sonja Date: Fri, 23 Jul 2021 15:34:41 +0200 Subject: [PATCH 15/70] Fixed mistakes in get_new_beta_and_l and started using it in BO --- sparseSpACE/HP_Optimization.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 6753cef..bcba5cb 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -4,6 +4,7 @@ import math import random from scipy.optimize import fmin +from scipy.optimize import minimize #performs Grid Optimization def perform_GO_classification(data): @@ -99,10 +100,8 @@ def perform_BO_classification(data, amt_it: int, dim: int): C_y[cur_amt_x] = perform_evaluation_at(data, new_x_rd[0], new_x_rd[1], new_x_rd[2], new_x_rd[3]) cur_amt_x += 1 else: - #erstelle neues beta und l - übergebe cur_length / amt_x damit cov stuff darin berechnet werden kann, C_x damit if in ev set geschaut werden kann - #erstelle neue cov matrix und vector mit neuem l - #berechne neuen wert mit neuem beta und l - #ggf ändere vorher schon cov matrix beim testen auf g? weil gebraucht wird für die berechnung + #erstelle neues beta und l und berechne neuen wert damit + get_new_beta_and_l(beta, cur_amt_x, new_x, C_x, C_y) print("Dead End! I am crying now") break for i in range(0, cur_amt_x, 1): @@ -185,16 +184,17 @@ def get_l_k(t: int): return 0.5 #TODO: get new beta and l. Currently mock function -def get_new_beta_and_l(cur_beta: float, cur_length, cur_x, C_x, C_y, cur_amt_x): +def get_new_beta_and_l(cur_beta: float, cur_amt_x, cur_x, C_x, C_y): #wsl fnct für get_x mit beta und l sinnvoll - auch für vorher #l nachher wieder zurücksetzen - bzw lt?? Was ist das, wie berechnet sich das? global l_k #if l_k should be changed permanently beta_h = 100 l_h = 20 - #0 if in ev set C_x, constant otherwise (3) - p = lambda x: check_if_in_array(x, C_x)*5 - #x[0] is \beta+d\beta, x[1] is l - new_x = lambda x: acq_x(x[1], x[0], C_x,C_y, cur_amt_x) + #0 if rd(x) is in ev set C_x, constant otherwise (5) + p = lambda x: check_if_in_array(round_x_classification(x), C_x)*5 + #gets the x value for certain l (z[1]) and beta (z[0]) + new_x = lambda z: acq_x(z[1], z[0], C_x,C_y, cur_amt_x) + #for g: x[0] is \beta+d\beta, x[1] is l. Also d\beta = \beta+d\beta-\beta g = lambda x: (x[0]-cur_beta)+np.linalg.norm(cur_x-new_x(x))+p(new_x(x)) bounds_g = ((cur_beta, beta_h), (l_k, l_h)) result = minimize(g, [1, 1], method='L-BFGS-B', bounds=bounds_g) From ced4912232a22ae107538ab332d1ee44e0d0e22e Mon Sep 17 00:00:00 2001 From: Sonja Date: Mon, 26 Jul 2021 15:24:40 +0200 Subject: [PATCH 16/70] Fixed some bugs, BUT: Cov matr is singular sometimes right now?? Should be impossible. Also get_beta_and_l is rarely used and has not worked yet --- sparseSpACE/HP_Optimization.py | 76 ++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index bcba5cb..cdc95a9 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -74,36 +74,39 @@ def perform_evaluation_at(cur_data, cur_lambd: float, cur_massl: bool, cur_min_l #TODO: will perform Bayesian Optimization; Currently basically performs BO with Naive rounding and a weird beta #Note: HP Input of classification is (lambd:float, massl:bool, min_lv:int <4, one_vs_others:bool) -def perform_BO_classification(data, amt_it: int, dim: int): +def perform_BO_classification(data, amt_it: int, dim: int, ev_is_rand: bool = True): #notes how many x values currently are in the evidence set - starts with amt_HP+1 amt_HP = 4 cur_amt_x: int = amt_HP+1 x_ret = None y_ret = None - print("I should do Bayesian Optimization for " + str(amt_it) + " iterations!") - C = create_evidence_set(data, amt_it, amt_HP) + print("I should do Bayesian Optimization for " + str(amt_it) + " iterations, ev_is_rand is " + str(ev_is_rand) + " !") + C = create_evidence_set(data, amt_it, amt_HP, ev_is_rand) C_x=C[0] + print("C_x: " + str(C_x)) C_y=C[1] - print("Evidence Set: \n" + str(C)) + print("C_y: " + str(C_y)) + #print("Evidence Set: \n" + str(C)) for i in range (0, amt_it, 1): print("iteration: " + str(i)) beta = get_beta(i+1) l = get_l_k(i) - print("beta_i: " + str(beta)) #value that will be evaluated and added to the evidence set - new_x=acq_x(l, beta, C_x, C_y, cur_amt_x) - print("new x: " + str(new_x) + " with function value: " + str(alpha(new_x))) + new_x=acq_x(beta, l, C_x, C_y, cur_amt_x) + print("new x: " + str(new_x)) new_x_rd = round_x_classification(new_x) - print("rounded: " + str(new_x_rd)) - if(not check_if_in_array(new_x_rd, C_x)): - C_x[cur_amt_x] = new_x_rd - C_y[cur_amt_x] = perform_evaluation_at(data, new_x_rd[0], new_x_rd[1], new_x_rd[2], new_x_rd[3]) - cur_amt_x += 1 - else: - #erstelle neues beta und l und berechne neuen wert damit - get_new_beta_and_l(beta, cur_amt_x, new_x, C_x, C_y) - print("Dead End! I am crying now") - break + print("new x rd: " + str(new_x_rd)) + #erstelle neues beta und l und berechne neuen wert damit + while(check_if_in_array(new_x_rd, C_x)): + print("!!!!!!!!!!!Need new beta and l") + beta_and_l = get_new_beta_and_l(beta, cur_amt_x, new_x, C_x, C_y) + new_x = acq_x(beta_and_l[0], beta_and_l[1], C_x, C_y, cur_amt_x) + new_x_rd = round_x_classification(new_x) + #break + C_x[cur_amt_x] = new_x_rd + C_y[cur_amt_x] = perform_evaluation_at(data, new_x_rd[0], new_x_rd[1], new_x_rd[2], new_x_rd[3]) + cur_amt_x += 1 + for i in range(0, cur_amt_x, 1): if(y_ret == None or y_ret1 - y[i] = perform_evaluation_at(data, new_x[0], new_x[1], new_x[2], new_x[3]) - #One could hard-code non-random values for testing + #y[i] = perform_evaluation_at(data, new_x[0], new_x[1], new_x[2], new_x[3]) + #needs to be casted to int bc otherwise it's automatically float which doesn't work + y[i] = perform_evaluation_at(data, x[i][0], int(x[i][1]), int(x[i][2]), int(x[i][3])) + else: + #hard-code non-random values for testing - current threw error at it. 24 + x[0] = [0.20213575, 0., 1., 0.] #[0.79125794, 0, 1, 0] + x[1] = [0.80125658, 0., 1., 0.] #[0.69819941, 1, 2, 0] + x[2] = [0.09898312, 1., 3., 1.] #[0.35823418, 0, 1, 0] + x[3] = [0.88249225, 1., 1., 0.] #[0.51043662, 1, 1, 0] + x[4] = [0.1321559, 1., 2., 1.] #[0.54776247, 0, 1, 0] + for i in range(0, dim_HP+1, 1): + y[i] = perform_evaluation_at(data, x[i][0], int(x[i][1]), int(x[i][2]), int(x[i][3])) return x, y #returns a random x for the given purpose @@ -185,15 +200,16 @@ def get_l_k(t: int): #TODO: get new beta and l. Currently mock function def get_new_beta_and_l(cur_beta: float, cur_amt_x, cur_x, C_x, C_y): + print("Getting new beta and l") #wsl fnct für get_x mit beta und l sinnvoll - auch für vorher #l nachher wieder zurücksetzen - bzw lt?? Was ist das, wie berechnet sich das? global l_k #if l_k should be changed permanently - beta_h = 100 - l_h = 20 + beta_h = 40 + l_h = 3 #0 if rd(x) is in ev set C_x, constant otherwise (5) p = lambda x: check_if_in_array(round_x_classification(x), C_x)*5 #gets the x value for certain l (z[1]) and beta (z[0]) - new_x = lambda z: acq_x(z[1], z[0], C_x,C_y, cur_amt_x) + new_x = lambda z: acq_x(z[0], z[1], C_x,C_y, cur_amt_x) #for g: x[0] is \beta+d\beta, x[1] is l. Also d\beta = \beta+d\beta-\beta g = lambda x: (x[0]-cur_beta)+np.linalg.norm(cur_x-new_x(x))+p(new_x(x)) bounds_g = ((cur_beta, beta_h), (l_k, l_h)) @@ -202,14 +218,14 @@ def get_new_beta_and_l(cur_beta: float, cur_amt_x, cur_x, C_x, C_y): return result #TODO acquire new x for l, beta, C_x, cur_amt_x, using GP-UCB -def acq_x(l: float, beta: float, C_x, C_y, cur_amt_x): - print("lol") - #wait no I need to change the l and the cov function D: +def acq_x(beta: float, l: float, C_x, C_y, cur_amt_x): global l_k old_l=l_k l_k=l K_matr = get_cov_matrix(C_x, cur_amt_x) - print(K_matr) + if(np.linalg.det(K_matr) == 0): + print("Covariance Matrix is indeed singular!") + #print(K_matr) mu = lambda x: get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(C_y)) sigma_sqrd = lambda x: k(x, x)-get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(get_cov_vector(C_x, x, cur_amt_x))) #takes sqrt of abs(sigma_sqrd) bc otherwise fmin gives an error - might be an overall warning sign tho @@ -218,7 +234,7 @@ def acq_x(l: float, beta: float, C_x, C_y, cur_amt_x): #negates alpha bc maximum has to be found alpha_neg = lambda x: -alpha(x) new_x=fmin(alpha_neg, [0, 0, 0, 0]) #Note: Vielleicht kann man die Auswertung bounden, damit keine Werte weit weg vom Möglichen rauskommen - print("new x: " + str(new_x) + " with function value: " + str(alpha(new_x))) + #print("new x: " + str(new_x) + " with function value: " + str(alpha(new_x))) return new_x def round_x_classification(x): @@ -249,6 +265,6 @@ def round_x_classification(x): data1 = deml.DataSet(sklearn_dataset, name='Input_Set') data_moons = data1.copy() data_moons.set_name('Moon_Set') -#perform_BO_classification(data_moons, 6, dimension) -#print(get_beta(1)) -perform_RO_classification(data_moons, 10, dimension) +#perform_evaluation_at(data_moons, 0.00242204, 0, 1, 0) #.98 evaluation! +perform_BO_classification(data_moons, 100, dimension, False) +#perform_RO_classification(data_moons, 10, dimension) From e44431085705b3e5b643876b269683ea866fcafb Mon Sep 17 00:00:00 2001 From: Sonja Date: Mon, 26 Jul 2021 18:52:56 +0200 Subject: [PATCH 17/70] Put bounds on finding the minimum of -alpha(x). Also corrected that only x value and not whole output of minimize is used --- sparseSpACE/HP_Optimization.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index cdc95a9..0a2c06b 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -183,7 +183,7 @@ def check_if_in_array(x, array): break return False -#TODO: will calculate and return current beta. t is current iteration +#calculates and returns current beta. t is current iteration def get_beta(t: int): r: float = 1.5 d: int = 4 @@ -201,9 +201,7 @@ def get_l_k(t: int): #TODO: get new beta and l. Currently mock function def get_new_beta_and_l(cur_beta: float, cur_amt_x, cur_x, C_x, C_y): print("Getting new beta and l") - #wsl fnct für get_x mit beta und l sinnvoll - auch für vorher - #l nachher wieder zurücksetzen - bzw lt?? Was ist das, wie berechnet sich das? - global l_k #if l_k should be changed permanently + global l_k beta_h = 40 l_h = 3 #0 if rd(x) is in ev set C_x, constant otherwise (5) @@ -213,7 +211,7 @@ def get_new_beta_and_l(cur_beta: float, cur_amt_x, cur_x, C_x, C_y): #for g: x[0] is \beta+d\beta, x[1] is l. Also d\beta = \beta+d\beta-\beta g = lambda x: (x[0]-cur_beta)+np.linalg.norm(cur_x-new_x(x))+p(new_x(x)) bounds_g = ((cur_beta, beta_h), (l_k, l_h)) - result = minimize(g, [1, 1], method='L-BFGS-B', bounds=bounds_g) + result = minimize(g, [1, 1], method='L-BFGS-B', bounds=bounds_g).x #result is in the form [new beta, new l] return result @@ -233,7 +231,9 @@ def acq_x(beta: float, l: float, C_x, C_y, cur_amt_x): alpha = lambda x: mu(x)+(math.sqrt(beta))*sigma(x) #negates alpha bc maximum has to be found alpha_neg = lambda x: -alpha(x) - new_x=fmin(alpha_neg, [0, 0, 0, 0]) #Note: Vielleicht kann man die Auswertung bounden, damit keine Werte weit weg vom Möglichen rauskommen + bounds_an = ((0, 3), (0, 1), (1, 3), (0, 1)) #bounds search space to vicinity of useful values + #problem: Nelder-Mead (same as fmin) cannot handle bounds -> use L-BFGS-B for now + new_x=minimize(alpha_neg, [0, 0, 0, 0], method='L-BFGS-B', bounds=bounds_an).x #print("new x: " + str(new_x) + " with function value: " + str(alpha(new_x))) return new_x @@ -266,5 +266,5 @@ def round_x_classification(x): data_moons = data1.copy() data_moons.set_name('Moon_Set') #perform_evaluation_at(data_moons, 0.00242204, 0, 1, 0) #.98 evaluation! -perform_BO_classification(data_moons, 100, dimension, False) +perform_BO_classification(data_moons, 5, dimension) #perform_RO_classification(data_moons, 10, dimension) From cafa73c17deac78e9a686ed85c7526972b9a822b Mon Sep 17 00:00:00 2001 From: Sonja Date: Mon, 26 Jul 2021 20:48:20 +0200 Subject: [PATCH 18/70] End of the day commit - still confused about the singular matrix --- sparseSpACE/HP_Optimization.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 0a2c06b..d6eee2c 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -99,10 +99,16 @@ def perform_BO_classification(data, amt_it: int, dim: int, ev_is_rand: bool = Tr #erstelle neues beta und l und berechne neuen wert damit while(check_if_in_array(new_x_rd, C_x)): print("!!!!!!!!!!!Need new beta and l") + old_x_rd = new_x_rd + print("old x rd was: " + str(old_x_rd)) beta_and_l = get_new_beta_and_l(beta, cur_amt_x, new_x, C_x, C_y) new_x = acq_x(beta_and_l[0], beta_and_l[1], C_x, C_y, cur_amt_x) new_x_rd = round_x_classification(new_x) - #break + print("new x rd is: " + str(new_x_rd)) + if(old_x_rd==new_x_rd): + print("We're in an infinite loop! Getting out") + break + print("adding " + str(new_x_rd)) C_x[cur_amt_x] = new_x_rd C_y[cur_amt_x] = perform_evaluation_at(data, new_x_rd[0], new_x_rd[1], new_x_rd[2], new_x_rd[3]) cur_amt_x += 1 @@ -126,7 +132,7 @@ def get_cov_matrix(x, cur_length: int): for j in range (0, cur_length, 1): K[i][j]=k(x[i], x[j]) if(np.linalg.det(K) == 0): - print("Oh no! The covariance matrix is singular!!") + print("Oh no! The covariance matrix is singular!! It is " + str(K)) return K #returns the vector k for a certain input new_x @@ -200,10 +206,11 @@ def get_l_k(t: int): #TODO: get new beta and l. Currently mock function def get_new_beta_and_l(cur_beta: float, cur_amt_x, cur_x, C_x, C_y): - print("Getting new beta and l") global l_k - beta_h = 40 - l_h = 3 + print("Getting new beta and l. Current beta: " + str(cur_beta) +", Current l: " + str(l_k)) + #making upper bounds dependable on current values so bounds are never too low + beta_h = cur_beta+50 + l_h = l_k+10 #0 if rd(x) is in ev set C_x, constant otherwise (5) p = lambda x: check_if_in_array(round_x_classification(x), C_x)*5 #gets the x value for certain l (z[1]) and beta (z[0]) @@ -266,5 +273,5 @@ def round_x_classification(x): data_moons = data1.copy() data_moons.set_name('Moon_Set') #perform_evaluation_at(data_moons, 0.00242204, 0, 1, 0) #.98 evaluation! -perform_BO_classification(data_moons, 5, dimension) +perform_BO_classification(data_moons, 100, dimension) #perform_RO_classification(data_moons, 10, dimension) From 2b2daedc33e1914793fd8a15cd009996b2fc9138 Mon Sep 17 00:00:00 2001 From: Sonja Date: Wed, 28 Jul 2021 14:38:29 +0200 Subject: [PATCH 19/70] Still don't know what I'm doing --- sparseSpACE/HP_Optimization.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index d6eee2c..918e950 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -107,11 +107,20 @@ def perform_BO_classification(data, amt_it: int, dim: int, ev_is_rand: bool = Tr print("new x rd is: " + str(new_x_rd)) if(old_x_rd==new_x_rd): print("We're in an infinite loop! Getting out") + #problem: after ending infinite loop the value is added, even though it was already in C_x + #..this makes the matrix singular. Shouldn't happen though with modified get_beta_and_l break + print("out of loop") print("adding " + str(new_x_rd)) C_x[cur_amt_x] = new_x_rd C_y[cur_amt_x] = perform_evaluation_at(data, new_x_rd[0], new_x_rd[1], new_x_rd[2], new_x_rd[3]) cur_amt_x += 1 + #ends everything if cov_matr is singular. Should this be dependant on l? Cov_matr being singular should not depend on l I think + if(np.linalg.det(get_cov_matrix(C_x, cur_amt_x)==0)): + print("With the nex x value added the Covariance Matrix is singular.\nBayesian Optimization can not be continued and will be ended here, after " + str(i) + " iterations") + break + else: + print("Apparently the cov_matr is not singular after iteration " + str(i)) for i in range(0, cur_amt_x, 1): if(y_ret == None or y_ret=1) #print(K_matr) mu = lambda x: get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(C_y)) sigma_sqrd = lambda x: k(x, x)-get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(get_cov_vector(C_x, x, cur_amt_x))) From 599fa3cf5a1b5014aadf06386b1a676cd5d003d7 Mon Sep 17 00:00:00 2001 From: Sonja Date: Wed, 28 Jul 2021 17:30:18 +0200 Subject: [PATCH 20/70] Iteration stops neatly if cov_matrix is singular --- sparseSpACE/HP_Optimization.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 918e950..fc4ad26 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -91,7 +91,13 @@ def perform_BO_classification(data, amt_it: int, dim: int, ev_is_rand: bool = Tr print("iteration: " + str(i)) beta = get_beta(i+1) l = get_l_k(i) + if(np.linalg.det(get_cov_matrix(C_x, cur_amt_x))==0): + print("With the nex x value added the Covariance Matrix is singular.\nBayesian Optimization can not be continued and will be ended here, at the start of iteration " + str(i)) + break + else: + print("Apparently the cov_matr is not singular at the beginning iteration " + str(i)) #value that will be evaluated and added to the evidence set + print("Getting new x:") new_x=acq_x(beta, l, C_x, C_y, cur_amt_x) print("new x: " + str(new_x)) new_x_rd = round_x_classification(new_x) @@ -116,7 +122,8 @@ def perform_BO_classification(data, amt_it: int, dim: int, ev_is_rand: bool = Tr C_y[cur_amt_x] = perform_evaluation_at(data, new_x_rd[0], new_x_rd[1], new_x_rd[2], new_x_rd[3]) cur_amt_x += 1 #ends everything if cov_matr is singular. Should this be dependant on l? Cov_matr being singular should not depend on l I think - if(np.linalg.det(get_cov_matrix(C_x, cur_amt_x)==0)): + print("Checking if cov_matr is singular") + if(np.linalg.det(get_cov_matrix(C_x, cur_amt_x)) == 0): print("With the nex x value added the Covariance Matrix is singular.\nBayesian Optimization can not be continued and will be ended here, after " + str(i) + " iterations") break else: @@ -172,11 +179,12 @@ def create_evidence_set(data, amt_it: int, dim_HP: int, is_random: bool = True): y[i] = perform_evaluation_at(data, x[i][0], int(x[i][1]), int(x[i][2]), int(x[i][3])) else: #hard-code non-random values for testing - current threw error at it. 24 + #use seed for random values instead of hard coded values? -> get used seed x[0] = [0.20213575, 0., 1., 0.] #[0.79125794, 0, 1, 0] x[1] = [0.80125658, 0., 1., 0.] #[0.69819941, 1, 2, 0] x[2] = [0.09898312, 1., 3., 1.] #[0.35823418, 0, 1, 0] x[3] = [0.88249225, 1., 1., 0.] #[0.51043662, 1, 1, 0] - x[4] = [0.1321559, 1., 2., 1.] #[0.54776247, 0, 1, 0] + x[4] = [0.1321559, 1., 2., 1.] #[0.54776247, 0, 1, 0] for i in range(0, dim_HP+1, 1): y[i] = perform_evaluation_at(data, x[i][0], int(x[i][1]), int(x[i][2]), int(x[i][3])) return x, y @@ -287,5 +295,5 @@ def round_x_classification(x): data_moons = data1.copy() data_moons.set_name('Moon_Set') #perform_evaluation_at(data_moons, 0.00242204, 0, 1, 0) #.98 evaluation! -perform_BO_classification(data_moons, 100, dimension) +perform_BO_classification(data_moons, 20, dimension) #perform_RO_classification(data_moons, 10, dimension) From d7cba48521e1eb7a5e3a476800b4a852696391d5 Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 3 Aug 2021 10:43:25 +0200 Subject: [PATCH 21/70] Changed DataSet, implemented another way to handle singular matrices (needed for min of g) --- ipynb/Tutorial_DEMachineLearning.ipynb | 2 +- sparseSpACE/HP_Optimization.py | 28 +++++++++++++++++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/ipynb/Tutorial_DEMachineLearning.ipynb b/ipynb/Tutorial_DEMachineLearning.ipynb index fb7936e..3297a61 100644 --- a/ipynb/Tutorial_DEMachineLearning.ipynb +++ b/ipynb/Tutorial_DEMachineLearning.ipynb @@ -222,7 +222,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.9" + "version": "3.8.5" } }, "nbformat": 4, diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index fc4ad26..ff8915d 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -94,8 +94,12 @@ def perform_BO_classification(data, amt_it: int, dim: int, ev_is_rand: bool = Tr if(np.linalg.det(get_cov_matrix(C_x, cur_amt_x))==0): print("With the nex x value added the Covariance Matrix is singular.\nBayesian Optimization can not be continued and will be ended here, at the start of iteration " + str(i)) break + if((get_cov_matrix(C_x, cur_amt_x) == np.identity(len(C_x))*(cur_amt_x+10)).all()): + print("cov_matr has been modified to be Id*scalar! This means it was singular -> ending at iteration " + str(i)) + break else: - print("Apparently the cov_matr is not singular at the beginning iteration " + str(i)) + print("Apparently the cov_matr is not singular at the beginning of iteration " + str(i)) + #value that will be evaluated and added to the evidence set print("Getting new x:") new_x=acq_x(beta, l, C_x, C_y, cur_amt_x) @@ -117,6 +121,9 @@ def perform_BO_classification(data, amt_it: int, dim: int, ev_is_rand: bool = Tr #..this makes the matrix singular. Shouldn't happen though with modified get_beta_and_l break print("out of loop") + if(old_x_rd==new_x_rd): + print("There was an infinite loop when getting new x. Ending BO at iteration " + str(i)) + break print("adding " + str(new_x_rd)) C_x[cur_amt_x] = new_x_rd C_y[cur_amt_x] = perform_evaluation_at(data, new_x_rd[0], new_x_rd[1], new_x_rd[2], new_x_rd[3]) @@ -126,6 +133,9 @@ def perform_BO_classification(data, amt_it: int, dim: int, ev_is_rand: bool = Tr if(np.linalg.det(get_cov_matrix(C_x, cur_amt_x)) == 0): print("With the nex x value added the Covariance Matrix is singular.\nBayesian Optimization can not be continued and will be ended here, after " + str(i) + " iterations") break + if((get_cov_matrix(C_x, cur_amt_x) == np.identity(len(C_x))*(cur_amt_x+10)).all()): + print("cov_matr has been modified to be Id*scalar! This means it was singular -> ending at iteration " + str(i)) + break else: print("Apparently the cov_matr is not singular after iteration " + str(i)) @@ -150,7 +160,11 @@ def get_cov_matrix(C_x, cur_length: int): if(np.linalg.det(K) == 0): print("Oh no! The covariance matrix is singular!! It is " + str(K)) print("Current C_x is: " + str(C_x)) - #K=None + #Maybe change matrix if it is singular? E.g. to scalar*Id, so it's def. not the smallest? + #Would have to deal with that in other methods though + #Important!! K is inverted. But also it's used some way in g?? Unsure what to do + #at least it doesn't crash rn + K = np.identity(len(C_x))*(cur_length+10) return K #returns the vector k for a certain input new_x @@ -227,6 +241,10 @@ def get_l_k(t: int): def get_new_beta_and_l(cur_beta: float, cur_amt_x, cur_x, C_x, C_y): global l_k print("Getting new beta and l. Current beta: " + str(cur_beta) +", Current l: " + str(l_k)) + if(np.linalg.det(get_cov_matrix(C_x, cur_amt_x))==0): + print("Getting new beta and l, but suddenly the cov matr is singular??") + else: + print("Apparently the cov_matr is not singular. Getting new beta and l") #making upper bounds dependable on current values so bounds are never too low beta_h = cur_beta+100 l_h = l_k+50 @@ -238,6 +256,8 @@ def get_new_beta_and_l(cur_beta: float, cur_amt_x, cur_x, C_x, C_y): g = lambda x: (x[0]-cur_beta)+np.linalg.norm(cur_x-new_x(x))+p(new_x(x)) bounds_g = ((cur_beta, beta_h), (l_k, l_h)) print("About to minimize g(...)") + #due to numerical inaccuracies a matrix might become singular with new beta and l + #(even though it wouldn't be) mathematically - how to deal with that?? result = minimize(g, [1, 1], method='L-BFGS-B', bounds=bounds_g).x print("New beta: " + str(result[0]) + ", New l: " + str(result[1])) #result is in the form [new beta, new l] @@ -294,6 +314,8 @@ def round_x_classification(x): data1 = deml.DataSet(sklearn_dataset, name='Input_Set') data_moons = data1.copy() data_moons.set_name('Moon_Set') +dataset_blobs = deml.datasets.make_blobs(n_samples=samples, n_features=dimension, centers=labels) +data_blobs = deml.DataSet(dataset_blobs, name='Blobs_Set') #perform_evaluation_at(data_moons, 0.00242204, 0, 1, 0) #.98 evaluation! -perform_BO_classification(data_moons, 20, dimension) +#perform_BO_classification(data_blobs, 20, dimension) #perform_RO_classification(data_moons, 10, dimension) From 9b44c37469d9b6a8171666261d6d0319b795d4d3 Mon Sep 17 00:00:00 2001 From: Sonja Date: Mon, 9 Aug 2021 19:48:10 +0200 Subject: [PATCH 22/70] Save before making big changes to make code more universally usable --- sparseSpACE/HP_Optimization.py | 65 ++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index ff8915d..52c7d2c 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -7,50 +7,47 @@ from scipy.optimize import minimize #performs Grid Optimization -def perform_GO_classification(data): +def perform_GO_classification(data, lambd_levels): #sklearn_dataset = deml.datasets.make_moons(n_samples=samples, noise=0.15) #data = deml.DataSet(sklearn_dataset, name='Input_Set') #for storing the current best evaluation and time best_evaluation = None - best_time = None + #best_time = None #storing the parameters of the best evaluation best_lambd = None best_masslump = None best_min_lv = None best_one_vs_others = None - #For Testing best evaluation: best_evaluation = 0.7 - #print("original best evaluation: " + str(best_evaluation)) - #todo: split Dataset + #todo: split Dataset?? #original classification: classification = deml.Classification(data, split_percentage=0.8, split_evenly=True, shuffle_data=True) #original settings: classification.perform_classification(masslumping=True, lambd=0.0, minimum_level=1, maximum_level=5, print_metrics=True) - #perform grid search for a few values of masslumping and lambd - lambd_levels = 10 - #Note: vllt besser logarithmisch, sehr kleine Werte oft besser (1/10, 1/100, 1/1000) - for cur_lambd in range (0, lambd_levels+1, 5): - #Note: lamd vor Allem sinnvoll wenn masslumping aus ist, sollte kaum unterschied machen wenn + #perform grid search for lambd_levels many lambd between 0 and 1 and all values for the rest + #Note: lambd vllt besser logarithmisch, sehr kleine Werte oft besser (1/10, 1/100, 1/1000) + #für for-schleife karthesisches produkt aus HO um alle tupel zu bilden und über diese tupel zu iterieren für beliebig dimensional + for cur_lambd_it in range (0, lambd_levels, 1): + cur_lambd = cur_lambd_it/lambd_levels + #Note: lambd vor Allem sinnvoll wenn masslumping aus ist, sollte kaum unterschied machen wenn an for cur_massl in [True, False]: - for cur_min_lv in range (1, 4, 1): + for cur_min_lv in range (1, 3, 1): for cur_one_vs_others in [True, False]: - classification = deml.Classification(data, split_percentage=0.8, split_evenly=True, shuffle_data=False) - print("current masslumping = " + str(cur_massl) + "; current lambd = " + str(cur_lambd/lambd_levels) + "; current min lv = " + str(cur_min_lv) + "; current one vs others = " + str(cur_one_vs_others)) - classification.perform_classification(masslumping=cur_massl, lambd=cur_lambd/lambd_levels, minimum_level=cur_min_lv, maximum_level=5, one_vs_others=cur_one_vs_others, print_metrics=False) - cur_time = classification._time_used - print ("current time needed = " + str(cur_time)) - evaluation = classification.evaluate() - print("Percentage of correct mappings",evaluation["Percentage correct"]) - #currently only looking at Percentage correct and time needed for evaluating - cur_evaluation = evaluation["Percentage correct"] - if best_evaluation == None or cur_evaluation > best_evaluation or (cur_evaluation == best_evaluation and (best_time == None or best_time>cur_time)): + #use "perform evaluation at" + #cur_time = classification._time_used + #print ("current time needed = " + str(cur_time)) + cur_evaluation = perform_evaluation_at(data, cur_lambd, cur_massl, cur_min_lv, cur_one_vs_others) + print("Percentage of correct mappings",cur_evaluation) + #if best_evaluation == None or cur_evaluation > best_evaluation or (cur_evaluation == best_evaluation and (best_time == None or best_time>cur_time)): + if(best_evaluation == None or cur_evaluation>best_evaluation): best_evaluation = cur_evaluation best_lambd = cur_lambd - best_time = cur_time + #best_time = cur_time0 best_masslump = cur_massl best_min_lv = cur_min_lv best_one_vs_others = cur_one_vs_others - print("Best evaluation is now " + str(best_evaluation) + " with masslump = " + str(best_masslump) + ", lambd = " + str(best_lambd) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others) + " and time = " + str(best_time)) - else: print("Best evaluation is still " + str(best_evaluation) + " with masslump = " + str(best_masslump) + ", lambd = " + str(best_lambd) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others) + " and time = " + str(best_time)) + print("Best evaluation is now " + str(best_evaluation) + " with lambd = " + str(best_lambd) + ", masslump = " + str(best_masslump) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others)) #+ " and time = " + str(best_time)) + else: print("Best evaluation is still " + str(best_evaluation) + " with lambd = " + str(best_lambd) + ", masslump = " + str(best_masslump) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others)) #+ " and time = " + str(best_time)) print ("END OF CURRENT EVALUATION \n") - print("In the end, best evaluation is " + str(best_evaluation) + " with masslump = " + str (best_masslump) + ", lamb = " + str(best_lambd) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others) + " and time = " + str(best_time)) + print("In the end, best evaluation is " + str(best_evaluation) + " with lambd = " + str(best_lambd) + ", masslump = " + str(best_masslump) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others)) + #+ " and time = " + str(best_time)) #performs random optimization def perform_RO_classification(data, amt_it: int, dim: int): @@ -70,13 +67,16 @@ def perform_evaluation_at(cur_data, cur_lambd: float, cur_massl: bool, cur_min_l classification.perform_classification(masslumping=cur_massl, lambd=cur_lambd, minimum_level=cur_min_lv, maximum_level=5, one_vs_others=cur_one_vs_others, print_metrics=False) evaluation = classification.evaluate() print("Percentage of correct mappings",evaluation["Percentage correct"]) + #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion return evaluation["Percentage correct"] +classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] + #TODO: will perform Bayesian Optimization; Currently basically performs BO with Naive rounding and a weird beta #Note: HP Input of classification is (lambd:float, massl:bool, min_lv:int <4, one_vs_others:bool) -def perform_BO_classification(data, amt_it: int, dim: int, ev_is_rand: bool = True): +def perform_BO_classification(data, amt_it: int, function = perform_evaluation_at, space = classification_space, ev_is_rand: bool = True): #notes how many x values currently are in the evidence set - starts with amt_HP+1 - amt_HP = 4 + amt_HP = len(space) cur_amt_x: int = amt_HP+1 x_ret = None y_ret = None @@ -164,6 +164,8 @@ def get_cov_matrix(C_x, cur_length: int): #Would have to deal with that in other methods though #Important!! K is inverted. But also it's used some way in g?? Unsure what to do #at least it doesn't crash rn + #add small values to diagonal? -> verfälscht sachen. Müsste relativ zu werten in Matrix angepasst werden + #z.b. 10^-15 oder so K = np.identity(len(C_x))*(cur_length+10) return K @@ -249,14 +251,15 @@ def get_new_beta_and_l(cur_beta: float, cur_amt_x, cur_x, C_x, C_y): beta_h = cur_beta+100 l_h = l_k+50 #0 if rd(x) is in ev set C_x, constant otherwise (5) + #penalty p(x) erhöhen wenn gleicher Wert rauskommt. z.B. immer +50 oder *2 oder anders exponentiell p = lambda x: check_if_in_array(round_x_classification(x), C_x)*50 #gets the x value for certain l (z[1]) and beta (z[0]) - new_x = lambda z: acq_x(z[0], z[1], C_x,C_y, cur_amt_x) + new_x = lambda z: acq_x(z[0], z[1], C_x, C_y, cur_amt_x) #for g: x[0] is \beta+d\beta, x[1] is l. Also d\beta = \beta+d\beta-\beta g = lambda x: (x[0]-cur_beta)+np.linalg.norm(cur_x-new_x(x))+p(new_x(x)) bounds_g = ((cur_beta, beta_h), (l_k, l_h)) print("About to minimize g(...)") - #due to numerical inaccuracies a matrix might become singular with new beta and l + #due to numerical inaccuracies a matrix might become singular with new beta and l #(even though it wouldn't be) mathematically - how to deal with that?? result = minimize(g, [1, 1], method='L-BFGS-B', bounds=bounds_g).x print("New beta: " + str(result[0]) + ", New l: " + str(result[1])) @@ -317,5 +320,7 @@ def round_x_classification(x): dataset_blobs = deml.datasets.make_blobs(n_samples=samples, n_features=dimension, centers=labels) data_blobs = deml.DataSet(dataset_blobs, name='Blobs_Set') #perform_evaluation_at(data_moons, 0.00242204, 0, 1, 0) #.98 evaluation! -#perform_BO_classification(data_blobs, 20, dimension) +#perform_BO_classification(data_blobs, 20) #perform_RO_classification(data_moons, 10, dimension) +#perform_GO_classification(data_moons, 20) +perform_evaluation_at(data_moons, 0.828603059876013, 0, 2, 1) From 18fe8a021b379aea7cc29461de25853384c68765 Mon Sep 17 00:00:00 2001 From: Sonja Date: Mon, 9 Aug 2021 22:32:36 +0200 Subject: [PATCH 23/70] Turned HP_O into a class - needs alterations though --- sparseSpACE/HP_Optimization.py | 578 +++++++++++++++++---------------- 1 file changed, 298 insertions(+), 280 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 52c7d2c..a901323 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -6,305 +6,313 @@ from scipy.optimize import fmin from scipy.optimize import minimize -#performs Grid Optimization -def perform_GO_classification(data, lambd_levels): - #sklearn_dataset = deml.datasets.make_moons(n_samples=samples, noise=0.15) - #data = deml.DataSet(sklearn_dataset, name='Input_Set') - #for storing the current best evaluation and time - best_evaluation = None - #best_time = None - #storing the parameters of the best evaluation - best_lambd = None - best_masslump = None - best_min_lv = None - best_one_vs_others = None - #todo: split Dataset?? - #original classification: classification = deml.Classification(data, split_percentage=0.8, split_evenly=True, shuffle_data=True) - #original settings: classification.perform_classification(masslumping=True, lambd=0.0, minimum_level=1, maximum_level=5, print_metrics=True) - #perform grid search for lambd_levels many lambd between 0 and 1 and all values for the rest - #Note: lambd vllt besser logarithmisch, sehr kleine Werte oft besser (1/10, 1/100, 1/1000) - #für for-schleife karthesisches produkt aus HO um alle tupel zu bilden und über diese tupel zu iterieren für beliebig dimensional - for cur_lambd_it in range (0, lambd_levels, 1): - cur_lambd = cur_lambd_it/lambd_levels - #Note: lambd vor Allem sinnvoll wenn masslumping aus ist, sollte kaum unterschied machen wenn an - for cur_massl in [True, False]: - for cur_min_lv in range (1, 3, 1): - for cur_one_vs_others in [True, False]: - #use "perform evaluation at" - #cur_time = classification._time_used - #print ("current time needed = " + str(cur_time)) - cur_evaluation = perform_evaluation_at(data, cur_lambd, cur_massl, cur_min_lv, cur_one_vs_others) - print("Percentage of correct mappings",cur_evaluation) - #if best_evaluation == None or cur_evaluation > best_evaluation or (cur_evaluation == best_evaluation and (best_time == None or best_time>cur_time)): - if(best_evaluation == None or cur_evaluation>best_evaluation): - best_evaluation = cur_evaluation - best_lambd = cur_lambd - #best_time = cur_time0 - best_masslump = cur_massl - best_min_lv = cur_min_lv - best_one_vs_others = cur_one_vs_others - print("Best evaluation is now " + str(best_evaluation) + " with lambd = " + str(best_lambd) + ", masslump = " + str(best_masslump) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others)) #+ " and time = " + str(best_time)) - else: print("Best evaluation is still " + str(best_evaluation) + " with lambd = " + str(best_lambd) + ", masslump = " + str(best_masslump) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others)) #+ " and time = " + str(best_time)) - print ("END OF CURRENT EVALUATION \n") - print("In the end, best evaluation is " + str(best_evaluation) + " with lambd = " + str(best_lambd) + ", masslump = " + str(best_masslump) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others)) - #+ " and time = " + str(best_time)) +#class HP_Optimization gets the data on which the ML Algo works, the function on which HP should be Optimized and search space for the HPs +#space is in form [["interval", , ], ["list", , , ,...], ....] +class HP_Optimization: + def __init__(self, data, function, hp_space): + self.data = data + self.function = function + self.hp_space = hp_space -#performs random optimization -def perform_RO_classification(data, amt_it: int, dim: int): - best_evaluation = 0 - best_x = [1, 1, 1, 1] - for i in range (0, amt_it, 1): - x = create_random_x() - new_eval = perform_evaluation_at(data, x[0], x[1], x[2], x[3]) - if new_eval>best_evaluation: - best_x = x - best_evaluation = new_eval - print("Best evaluation in " + str(amt_it) + " random steps: " + str(best_evaluation) + " at " + str(x)) + #performs Grid Optimization + def perform_GO_classification(self, data, lambd_levels): + #sklearn_dataset = deml.datasets.make_moons(n_samples=samples, noise=0.15) + #data = deml.DataSet(sklearn_dataset, name='Input_Set') + #for storing the current best evaluation and time + best_evaluation = None + #best_time = None + #storing the parameters of the best evaluation + best_lambd = None + best_masslump = None + best_min_lv = None + best_one_vs_others = None + #todo: split Dataset?? + #original classification: classification = deml.Classification(data, split_percentage=0.8, split_evenly=True, shuffle_data=True) + #original settings: classification.perform_classification(masslumping=True, lambd=0.0, minimum_level=1, maximum_level=5, print_metrics=True) + #perform grid search for lambd_levels many lambd between 0 and 1 and all values for the rest + #Note: lambd vllt besser logarithmisch, sehr kleine Werte oft besser (1/10, 1/100, 1/1000) + #für for-schleife karthesisches produkt aus HO um alle tupel zu bilden und über diese tupel zu iterieren für beliebig dimensional + for cur_lambd_it in range (0, lambd_levels, 1): + cur_lambd = cur_lambd_it/lambd_levels + #Note: lambd vor Allem sinnvoll wenn masslumping aus ist, sollte kaum unterschied machen wenn an + for cur_massl in [True, False]: + for cur_min_lv in range (1, 3, 1): + for cur_one_vs_others in [True, False]: + #use "perform evaluation at" + #cur_time = classification._time_used + #print ("current time needed = " + str(cur_time)) + cur_evaluation = perform_evaluation_at(data, cur_lambd, cur_massl, cur_min_lv, cur_one_vs_others) + print("Percentage of correct mappings",cur_evaluation) + #if best_evaluation == None or cur_evaluation > best_evaluation or (cur_evaluation == best_evaluation and (best_time == None or best_time>cur_time)): + if(best_evaluation == None or cur_evaluation>best_evaluation): + best_evaluation = cur_evaluation + best_lambd = cur_lambd + #best_time = cur_time0 + best_masslump = cur_massl + best_min_lv = cur_min_lv + best_one_vs_others = cur_one_vs_others + print("Best evaluation is now " + str(best_evaluation) + " with lambd = " + str(best_lambd) + ", masslump = " + str(best_masslump) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others)) #+ " and time = " + str(best_time)) + else: print("Best evaluation is still " + str(best_evaluation) + " with lambd = " + str(best_lambd) + ", masslump = " + str(best_masslump) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others)) #+ " and time = " + str(best_time)) + print ("END OF CURRENT EVALUATION \n") + print("In the end, best evaluation is " + str(best_evaluation) + " with lambd = " + str(best_lambd) + ", masslump = " + str(best_masslump) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others)) + #+ " and time = " + str(best_time)) -#returns evaluation for certain Parameters on a certain data set -def perform_evaluation_at(cur_data, cur_lambd: float, cur_massl: bool, cur_min_lv: int, cur_one_vs_others: bool): - classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) - classification.perform_classification(masslumping=cur_massl, lambd=cur_lambd, minimum_level=cur_min_lv, maximum_level=5, one_vs_others=cur_one_vs_others, print_metrics=False) - evaluation = classification.evaluate() - print("Percentage of correct mappings",evaluation["Percentage correct"]) - #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion - return evaluation["Percentage correct"] + #performs random optimization + def perform_RO_classification(self, data, amt_it: int, dim: int): + best_evaluation = 0 + best_x = [1, 1, 1, 1] + for i in range (0, amt_it, 1): + x = create_random_x() + new_eval = perform_evaluation_at(data, x[0], x[1], x[2], x[3]) + if new_eval>best_evaluation: + best_x = x + best_evaluation = new_eval + print("Best evaluation in " + str(amt_it) + " random steps: " + str(best_evaluation) + " at " + str(x)) -classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] + #returns evaluation for certain Parameters on a certain data set + def perform_evaluation_at(self, cur_data, cur_lambd: float, cur_massl: bool, cur_min_lv: int, cur_one_vs_others: bool): + classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) + classification.perform_classification(masslumping=cur_massl, lambd=cur_lambd, minimum_level=cur_min_lv, maximum_level=5, one_vs_others=cur_one_vs_others, print_metrics=False) + evaluation = classification.evaluate() + print("Percentage of correct mappings",evaluation["Percentage correct"]) + #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion + return evaluation["Percentage correct"] -#TODO: will perform Bayesian Optimization; Currently basically performs BO with Naive rounding and a weird beta -#Note: HP Input of classification is (lambd:float, massl:bool, min_lv:int <4, one_vs_others:bool) -def perform_BO_classification(data, amt_it: int, function = perform_evaluation_at, space = classification_space, ev_is_rand: bool = True): - #notes how many x values currently are in the evidence set - starts with amt_HP+1 - amt_HP = len(space) - cur_amt_x: int = amt_HP+1 - x_ret = None - y_ret = None - print("I should do Bayesian Optimization for " + str(amt_it) + " iterations, ev_is_rand is " + str(ev_is_rand) + " !") - C = create_evidence_set(data, amt_it, amt_HP, ev_is_rand) - C_x=C[0] - print("C_x: " + str(C_x)) - C_y=C[1] - print("C_y: " + str(C_y)) - #print("Evidence Set: \n" + str(C)) - for i in range (0, amt_it, 1): - print("iteration: " + str(i)) - beta = get_beta(i+1) - l = get_l_k(i) - if(np.linalg.det(get_cov_matrix(C_x, cur_amt_x))==0): - print("With the nex x value added the Covariance Matrix is singular.\nBayesian Optimization can not be continued and will be ended here, at the start of iteration " + str(i)) - break - if((get_cov_matrix(C_x, cur_amt_x) == np.identity(len(C_x))*(cur_amt_x+10)).all()): - print("cov_matr has been modified to be Id*scalar! This means it was singular -> ending at iteration " + str(i)) - break - else: - print("Apparently the cov_matr is not singular at the beginning of iteration " + str(i)) + classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] - #value that will be evaluated and added to the evidence set - print("Getting new x:") - new_x=acq_x(beta, l, C_x, C_y, cur_amt_x) - print("new x: " + str(new_x)) - new_x_rd = round_x_classification(new_x) - print("new x rd: " + str(new_x_rd)) - #erstelle neues beta und l und berechne neuen wert damit - while(check_if_in_array(new_x_rd, C_x)): - print("!!!!!!!!!!!Need new beta and l") - old_x_rd = new_x_rd - print("old x rd was: " + str(old_x_rd)) - beta_and_l = get_new_beta_and_l(beta, cur_amt_x, new_x, C_x, C_y) - new_x = acq_x(beta_and_l[0], beta_and_l[1], C_x, C_y, cur_amt_x) + #TODO: will perform Bayesian Optimization; Currently basically performs BO with Naive rounding and a weird beta + #Note: HP Input of classification is (lambd:float, massl:bool, min_lv:int <4, one_vs_others:bool) + def perform_BO_classification(self, data, amt_it: int, function = perform_evaluation_at, space = classification_space, ev_is_rand: bool = True): + #notes how many x values currently are in the evidence set - starts with amt_HP+1 + amt_HP = len(space) + cur_amt_x: int = amt_HP+1 + x_ret = None + y_ret = None + print("I should do Bayesian Optimization for " + str(amt_it) + " iterations, ev_is_rand is " + str(ev_is_rand) + " !") + C = create_evidence_set(data, amt_it, amt_HP, ev_is_rand) + C_x=C[0] + print("C_x: " + str(C_x)) + C_y=C[1] + print("C_y: " + str(C_y)) + #print("Evidence Set: \n" + str(C)) + for i in range (0, amt_it, 1): + print("iteration: " + str(i)) + beta = get_beta(i+1) + l = get_l_k(i) + if(np.linalg.det(get_cov_matrix(C_x, cur_amt_x))==0): + print("With the nex x value added the Covariance Matrix is singular.\nBayesian Optimization can not be continued and will be ended here, at the start of iteration " + str(i)) + break + if((get_cov_matrix(C_x, cur_amt_x) == np.identity(len(C_x))*(cur_amt_x+10)).all()): + print("cov_matr has been modified to be Id*scalar! This means it was singular -> ending at iteration " + str(i)) + break + else: + print("Apparently the cov_matr is not singular at the beginning of iteration " + str(i)) + + #value that will be evaluated and added to the evidence set + print("Getting new x:") + new_x=acq_x(beta, l, C_x, C_y, cur_amt_x) + print("new x: " + str(new_x)) new_x_rd = round_x_classification(new_x) - print("new x rd is: " + str(new_x_rd)) + print("new x rd: " + str(new_x_rd)) + #erstelle neues beta und l und berechne neuen wert damit + while(check_if_in_array(new_x_rd, C_x)): + print("!!!!!!!!!!!Need new beta and l") + old_x_rd = new_x_rd + print("old x rd was: " + str(old_x_rd)) + beta_and_l = get_new_beta_and_l(beta, cur_amt_x, new_x, C_x, C_y) + new_x = acq_x(beta_and_l[0], beta_and_l[1], C_x, C_y, cur_amt_x) + new_x_rd = round_x_classification(new_x) + print("new x rd is: " + str(new_x_rd)) + if(old_x_rd==new_x_rd): + print("We're in an infinite loop! Getting out") + #problem: after ending infinite loop the value is added, even though it was already in C_x + #..this makes the matrix singular. Shouldn't happen though with modified get_beta_and_l + break + print("out of loop") if(old_x_rd==new_x_rd): - print("We're in an infinite loop! Getting out") - #problem: after ending infinite loop the value is added, even though it was already in C_x - #..this makes the matrix singular. Shouldn't happen though with modified get_beta_and_l + print("There was an infinite loop when getting new x. Ending BO at iteration " + str(i)) break - print("out of loop") - if(old_x_rd==new_x_rd): - print("There was an infinite loop when getting new x. Ending BO at iteration " + str(i)) - break - print("adding " + str(new_x_rd)) - C_x[cur_amt_x] = new_x_rd - C_y[cur_amt_x] = perform_evaluation_at(data, new_x_rd[0], new_x_rd[1], new_x_rd[2], new_x_rd[3]) - cur_amt_x += 1 - #ends everything if cov_matr is singular. Should this be dependant on l? Cov_matr being singular should not depend on l I think - print("Checking if cov_matr is singular") - if(np.linalg.det(get_cov_matrix(C_x, cur_amt_x)) == 0): - print("With the nex x value added the Covariance Matrix is singular.\nBayesian Optimization can not be continued and will be ended here, after " + str(i) + " iterations") - break - if((get_cov_matrix(C_x, cur_amt_x) == np.identity(len(C_x))*(cur_amt_x+10)).all()): - print("cov_matr has been modified to be Id*scalar! This means it was singular -> ending at iteration " + str(i)) - break - else: - print("Apparently the cov_matr is not singular after iteration " + str(i)) + print("adding " + str(new_x_rd)) + C_x[cur_amt_x] = new_x_rd + C_y[cur_amt_x] = perform_evaluation_at(data, new_x_rd[0], new_x_rd[1], new_x_rd[2], new_x_rd[3]) + cur_amt_x += 1 + #ends everything if cov_matr is singular. Should this be dependant on l? Cov_matr being singular should not depend on l I think + print("Checking if cov_matr is singular") + if(np.linalg.det(get_cov_matrix(C_x, cur_amt_x)) == 0): + print("With the nex x value added the Covariance Matrix is singular.\nBayesian Optimization can not be continued and will be ended here, after " + str(i) + " iterations") + break + if((get_cov_matrix(C_x, cur_amt_x) == np.identity(len(C_x))*(cur_amt_x+10)).all()): + print("cov_matr has been modified to be Id*scalar! This means it was singular -> ending at iteration " + str(i)) + break + else: + print("Apparently the cov_matr is not singular after iteration " + str(i)) - for i in range(0, cur_amt_x, 1): - if(y_ret == None or y_ret verfälscht sachen. Müsste relativ zu werten in Matrix angepasst werden - #z.b. 10^-15 oder so - K = np.identity(len(C_x))*(cur_length+10) - return K + #cur_length is needed so that rest of matrix stays Id + def get_cov_matrix(self, C_x, cur_length: int): + K = np.identity(len(C_x)) + for i in range (0, cur_length, 1): + for j in range (0, cur_length, 1): + K[i][j]=k(C_x[i], C_x[j]) + if(np.linalg.det(K) == 0): + print("Oh no! The covariance matrix is singular!! It is " + str(K)) + print("Current C_x is: " + str(C_x)) + #Maybe change matrix if it is singular? E.g. to scalar*Id, so it's def. not the smallest? + #Would have to deal with that in other methods though + #Important!! K is inverted. But also it's used some way in g?? Unsure what to do + #at least it doesn't crash rn + #add small values to diagonal? -> verfälscht sachen. Müsste relativ zu werten in Matrix angepasst werden + #z.b. 10^-15 oder so + K = np.identity(len(C_x))*(cur_length+10) + return K -#returns the vector k for a certain input new_x -def get_cov_vector(ev_x, new_x, cur_length: int): - #somehow the rest of the function would still work even if x is wrong size, but no idea why / what it does. Better to check - if(len(ev_x[0]) != len(new_x)): - print("Input x has the wrong size") - return None - k_vec = np.zeros(len(ev_x)) - for i in range (0, cur_length, 1): - k_vec[i] = k(ev_x[i], new_x) - return k_vec + #returns the vector k for a certain input new_x + def get_cov_vector(self, ev_x, new_x, cur_length: int): + #somehow the rest of the function would still work even if x is wrong size, but no idea why / what it does. Better to check + if(len(ev_x[0]) != len(new_x)): + print("Input x has the wrong size") + return None + k_vec = np.zeros(len(ev_x)) + for i in range (0, cur_length, 1): + k_vec[i] = k(ev_x[i], new_x) + return k_vec -def create_evidence_set(data, amt_it: int, dim_HP: int, is_random: bool = True): - x = np.zeros(shape=(amt_it+dim_HP+1, dim_HP)) - y = np.zeros((amt_it+dim_HP+1)) - if(is_random): - for i in range(0, dim_HP+1, 1): - new_x = create_random_x() - while(check_if_in_array(new_x, x)): + def create_evidence_set(self, data, amt_it: int, dim_HP: int, is_random: bool = True): + x = np.zeros(shape=(amt_it+dim_HP+1, dim_HP)) + y = np.zeros((amt_it+dim_HP+1)) + if(is_random): + for i in range(0, dim_HP+1, 1): new_x = create_random_x() - x[i] = new_x - #evaluating takes quite some time, especially for min_lv>1 - #y[i] = perform_evaluation_at(data, new_x[0], new_x[1], new_x[2], new_x[3]) - #needs to be casted to int bc otherwise it's automatically float which doesn't work - y[i] = perform_evaluation_at(data, x[i][0], int(x[i][1]), int(x[i][2]), int(x[i][3])) - else: - #hard-code non-random values for testing - current threw error at it. 24 - #use seed for random values instead of hard coded values? -> get used seed - x[0] = [0.20213575, 0., 1., 0.] #[0.79125794, 0, 1, 0] - x[1] = [0.80125658, 0., 1., 0.] #[0.69819941, 1, 2, 0] - x[2] = [0.09898312, 1., 3., 1.] #[0.35823418, 0, 1, 0] - x[3] = [0.88249225, 1., 1., 0.] #[0.51043662, 1, 1, 0] - x[4] = [0.1321559, 1., 2., 1.] #[0.54776247, 0, 1, 0] - for i in range(0, dim_HP+1, 1): - y[i] = perform_evaluation_at(data, x[i][0], int(x[i][1]), int(x[i][2]), int(x[i][3])) - return x, y + while(check_if_in_array(new_x, x)): + new_x = create_random_x() + x[i] = new_x + #evaluating takes quite some time, especially for min_lv>1 + #y[i] = perform_evaluation_at(data, new_x[0], new_x[1], new_x[2], new_x[3]) + #needs to be casted to int bc otherwise it's automatically float which doesn't work + y[i] = perform_evaluation_at(data, x[i][0], int(x[i][1]), int(x[i][2]), int(x[i][3])) + else: + #hard-code non-random values for testing - current threw error at it. 24 + #use seed for random values instead of hard coded values? -> get used seed + x[0] = [0.20213575, 0., 1., 0.] #[0.79125794, 0, 1, 0] + x[1] = [0.80125658, 0., 1., 0.] #[0.69819941, 1, 2, 0] + x[2] = [0.09898312, 1., 3., 1.] #[0.35823418, 0, 1, 0] + x[3] = [0.88249225, 1., 1., 0.] #[0.51043662, 1, 1, 0] + x[4] = [0.1321559, 1., 2., 1.] #[0.54776247, 0, 1, 0] + for i in range(0, dim_HP+1, 1): + y[i] = perform_evaluation_at(data, x[i][0], int(x[i][1]), int(x[i][2]), int(x[i][3])) + return x, y -#returns a random x for the given purpose -def create_random_x(purpose = "classification"): - if(purpose == "classification"): - return [random.random(), random.randint(0, 1), random.randint(1, 3), random.randint(0, 1)] - else: - print("Invalid Input") + #returns a random x for the given purpose + def create_random_x(self, purpose = "classification"): + if(purpose == "classification"): + return [random.random(), random.randint(0, 1), random.randint(1, 3), random.randint(0, 1)] + else: + print("Invalid Input") -#checks if array has the element x, returns True if yes. Caution: If x has same length but different dimension to the elements of array this may give unintended results -#maybe later a threshhold can be added if only part of the matrix should be checked -def check_if_in_array(x, array): - if(len(x) != len(array[0])): - #print("not same size") - return False - for i in range(0, len(array), 1): - if((x == array[i]).all()): - return True - break - return False + #checks if array has the element x, returns True if yes. Caution: If x has same length but different dimension to the elements of array this may give unintended results + #maybe later a threshhold can be added if only part of the matrix should be checked + def check_if_in_array(self, x, array): + if(len(x) != len(array[0])): + #print("not same size") + return False + for i in range(0, len(array), 1): + if((x == array[i]).all()): + return True + break + return False -#calculates and returns current beta. t is current iteration -def get_beta(t: int): - r: float = 1.5 - d: int = 4 - delta: float = 0.1 - a: float = 1 - b: float = 1 - beta = 2*math.log(t**2*2*math.pi**2/3*delta)+2*d*math.log(t**2*d*b*r*math.sqrt(math.log(4*d*a/delta))) - print(beta) - return beta + #calculates and returns current beta. t is current iteration + def get_beta(self, t: int): + r: float = 1.5 + d: int = 4 + delta: float = 0.1 + a: float = 1 + b: float = 1 + beta = 2*math.log(t**2*2*math.pi**2/3*delta)+2*d*math.log(t**2*d*b*r*math.sqrt(math.log(4*d*a/delta))) + print(beta) + return beta -#TODO: get l_k and use it -def get_l_k(t: int): - return 0.5 + #TODO: get l_k and use it + def get_l_k(self, t: int): + return 0.5 -#TODO: get new beta and l. Currently mock function -def get_new_beta_and_l(cur_beta: float, cur_amt_x, cur_x, C_x, C_y): - global l_k - print("Getting new beta and l. Current beta: " + str(cur_beta) +", Current l: " + str(l_k)) - if(np.linalg.det(get_cov_matrix(C_x, cur_amt_x))==0): - print("Getting new beta and l, but suddenly the cov matr is singular??") - else: - print("Apparently the cov_matr is not singular. Getting new beta and l") - #making upper bounds dependable on current values so bounds are never too low - beta_h = cur_beta+100 - l_h = l_k+50 - #0 if rd(x) is in ev set C_x, constant otherwise (5) - #penalty p(x) erhöhen wenn gleicher Wert rauskommt. z.B. immer +50 oder *2 oder anders exponentiell - p = lambda x: check_if_in_array(round_x_classification(x), C_x)*50 - #gets the x value for certain l (z[1]) and beta (z[0]) - new_x = lambda z: acq_x(z[0], z[1], C_x, C_y, cur_amt_x) - #for g: x[0] is \beta+d\beta, x[1] is l. Also d\beta = \beta+d\beta-\beta - g = lambda x: (x[0]-cur_beta)+np.linalg.norm(cur_x-new_x(x))+p(new_x(x)) - bounds_g = ((cur_beta, beta_h), (l_k, l_h)) - print("About to minimize g(...)") - #due to numerical inaccuracies a matrix might become singular with new beta and l - #(even though it wouldn't be) mathematically - how to deal with that?? - result = minimize(g, [1, 1], method='L-BFGS-B', bounds=bounds_g).x - print("New beta: " + str(result[0]) + ", New l: " + str(result[1])) - #result is in the form [new beta, new l] - return result + #TODO: get new beta and l. Currently mock function + def get_new_beta_and_l(self, cur_beta: float, cur_amt_x, cur_x, C_x, C_y): + global l_k + print("Getting new beta and l. Current beta: " + str(cur_beta) +", Current l: " + str(l_k)) + if(np.linalg.det(get_cov_matrix(C_x, cur_amt_x))==0): + print("Getting new beta and l, but suddenly the cov matr is singular??") + else: + print("Apparently the cov_matr is not singular. Getting new beta and l") + #making upper bounds dependable on current values so bounds are never too low + beta_h = cur_beta+100 + l_h = l_k+50 + #0 if rd(x) is in ev set C_x, constant otherwise (5) + #penalty p(x) erhöhen wenn gleicher Wert rauskommt. z.B. immer +50 oder *2 oder anders exponentiell + p = lambda x: check_if_in_array(round_x_classification(x), C_x)*50 + #gets the x value for certain l (z[1]) and beta (z[0]) + new_x = lambda z: acq_x(z[0], z[1], C_x, C_y, cur_amt_x) + #for g: x[0] is \beta+d\beta, x[1] is l. Also d\beta = \beta+d\beta-\beta + g = lambda x: (x[0]-cur_beta)+np.linalg.norm(cur_x-new_x(x))+p(new_x(x)) + bounds_g = ((cur_beta, beta_h), (l_k, l_h)) + print("About to minimize g(...)") + #due to numerical inaccuracies a matrix might become singular with new beta and l + #(even though it wouldn't be) mathematically - how to deal with that?? + result = minimize(g, [1, 1], method='L-BFGS-B', bounds=bounds_g).x + print("New beta: " + str(result[0]) + ", New l: " + str(result[1])) + #result is in the form [new beta, new l] + return result -#TODO acquire new x for l, beta, C_x, cur_amt_x, using GP-UCB -def acq_x(beta: float, l: float, C_x, C_y, cur_amt_x): - global l_k - old_l=l_k - l_k=l - K_matr = get_cov_matrix(C_x, cur_amt_x) - if(np.linalg.det(K_matr) == 0): - print("Covariance Matrix is indeed singular!") - #return a value that's bullshit? like [0, 0, 0, 0] (min lv is >=1) - #print(K_matr) - mu = lambda x: get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(C_y)) - sigma_sqrd = lambda x: k(x, x)-get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(get_cov_vector(C_x, x, cur_amt_x))) - #takes sqrt of abs(sigma_sqrd) bc otherwise fmin gives an error - might be an overall warning sign tho - sigma = lambda x: math.sqrt(abs(sigma_sqrd(x))) - alpha = lambda x: mu(x)+(math.sqrt(beta))*sigma(x) - #negates alpha bc maximum has to be found - alpha_neg = lambda x: -alpha(x) - bounds_an = ((0, 3), (0, 1), (1, 3), (0, 1)) #bounds search space to vicinity of useful values - #problem: Nelder-Mead (same as fmin) cannot handle bounds -> use L-BFGS-B for now - new_x=minimize(alpha_neg, [0, 0, 0, 0], method='L-BFGS-B', bounds=bounds_an).x - #print("new x: " + str(new_x) + " with function value: " + str(alpha(new_x))) - return new_x + #TODO acquire new x for l, beta, C_x, cur_amt_x, using GP-UCB + def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): + global l_k + old_l=l_k + l_k=l + K_matr = get_cov_matrix(C_x, cur_amt_x) + if(np.linalg.det(K_matr) == 0): + print("Covariance Matrix is indeed singular!") + #return a value that's bullshit? like [0, 0, 0, 0] (min lv is >=1) + #print(K_matr) + mu = lambda x: get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(C_y)) + sigma_sqrd = lambda x: k(x, x)-get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(get_cov_vector(C_x, x, cur_amt_x))) + #takes sqrt of abs(sigma_sqrd) bc otherwise fmin gives an error - might be an overall warning sign tho + sigma = lambda x: math.sqrt(abs(sigma_sqrd(x))) + alpha = lambda x: mu(x)+(math.sqrt(beta))*sigma(x) + #negates alpha bc maximum has to be found + alpha_neg = lambda x: -alpha(x) + bounds_an = ((0, 3), (0, 1), (1, 3), (0, 1)) #bounds search space to vicinity of useful values + #problem: Nelder-Mead (same as fmin) cannot handle bounds -> use L-BFGS-B for now + new_x=minimize(alpha_neg, [0, 0, 0, 0], method='L-BFGS-B', bounds=bounds_an).x + #print("new x: " + str(new_x) + " with function value: " + str(alpha(new_x))) + return new_x -def round_x_classification(x): - if len(x) < 4: - print("Input too short! Returning default values") - return [0, 0, 1, 0] - if len(x) > 4: - print("Input too long! Cropping") - #rounds the values of new_x to values usable as HPs - the question is what kind of rounding makes sense - #e.g if lambda should be bounded or more values should be rounded to 0 - lambd = x[0] - massl = math.trunc(x[1]) #is true if abs val > 1 - min_lv = math.trunc(x[2]) - if(min_lv<1): min_lv = 1 - elif (min_lv>3): min_lv = 3 - one_vs_others = math.trunc(x[3]) - new_x_rd = [lambd, massl, min_lv, one_vs_others] - return new_x_rd + def round_x_classification(self, x): + if len(x) < 4: + print("Input too short! Returning default values") + return [0, 0, 1, 0] + if len(x) > 4: + print("Input too long! Cropping") + #rounds the values of new_x to values usable as HPs - the question is what kind of rounding makes sense + #e.g if lambda should be bounded or more values should be rounded to 0 + lambd = x[0] + massl = math.trunc(x[1]) #is true if abs val > 1 + min_lv = math.trunc(x[2]) + if(min_lv<1): min_lv = 1 + elif (min_lv>3): min_lv = 3 + one_vs_others = math.trunc(x[3]) + new_x_rd = [lambd, massl, min_lv, one_vs_others] + return new_x_rd #Basically copied code from Tutorial_DEMachineLearning #prepare Dataset @@ -323,4 +331,14 @@ def round_x_classification(x): #perform_BO_classification(data_blobs, 20) #perform_RO_classification(data_moons, 10, dimension) #perform_GO_classification(data_moons, 20) -perform_evaluation_at(data_moons, 0.828603059876013, 0, 2, 1) +#perform_evaluation_at(self, data_moons, 0.828603059876013, 0, 2, 1) +def perform_evaluation_at(cur_data, cur_lambd: float, cur_massl: bool, cur_min_lv: int, cur_one_vs_others: bool): + classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) + classification.perform_classification(masslumping=cur_massl, lambd=cur_lambd, minimum_level=cur_min_lv, maximum_level=5, one_vs_others=cur_one_vs_others, print_metrics=False) + evaluation = classification.evaluate() + print("Percentage of correct mappings",evaluation["Percentage correct"]) + #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion + return evaluation["Percentage correct"] +classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] +HPO = HP_Optimization(data_moons, perform_evaluation_at, classification_space) +HPO.perform_evaluation_at(data_moons, 0.00242204, 0, 1, 0) From e4c195285aa2ca1fe0146f74b2fcf376733eadf3 Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 10 Aug 2021 09:40:43 +0200 Subject: [PATCH 24/70] started using given function --- sparseSpACE/HP_Optimization.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index a901323..b10052b 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -6,11 +6,11 @@ from scipy.optimize import fmin from scipy.optimize import minimize -#class HP_Optimization gets the data on which the ML Algo works, the function on which HP should be Optimized and search space for the HPs +#class HP_Optimization gets (data not needed bc only pea needs that which is the function!), +#the function on which HP should be Optimized and search space for the HPs #space is in form [["interval", , ], ["list", , , ,...], ....] class HP_Optimization: - def __init__(self, data, function, hp_space): - self.data = data + def __init__(self, function, hp_space): self.function = function self.hp_space = hp_space @@ -71,12 +71,13 @@ def perform_RO_classification(self, data, amt_it: int, dim: int): #returns evaluation for certain Parameters on a certain data set def perform_evaluation_at(self, cur_data, cur_lambd: float, cur_massl: bool, cur_min_lv: int, cur_one_vs_others: bool): - classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) - classification.perform_classification(masslumping=cur_massl, lambd=cur_lambd, minimum_level=cur_min_lv, maximum_level=5, one_vs_others=cur_one_vs_others, print_metrics=False) - evaluation = classification.evaluate() - print("Percentage of correct mappings",evaluation["Percentage correct"]) - #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion - return evaluation["Percentage correct"] + #classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) + #classification.perform_classification(masslumping=cur_massl, lambd=cur_lambd, minimum_level=cur_min_lv, maximum_level=5, one_vs_others=cur_one_vs_others, print_metrics=False) + #evaluation = classification.evaluate() + #print("Percentage of correct mappings",evaluation["Percentage correct"]) + ##wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion + #return evaluation["Percentage correct"] + return self.function(cur_lambd, cur_massl, cur_min_lv, cur_one_vs_others) classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] @@ -332,7 +333,11 @@ def round_x_classification(self, x): #perform_RO_classification(data_moons, 10, dimension) #perform_GO_classification(data_moons, 20) #perform_evaluation_at(self, data_moons, 0.828603059876013, 0, 2, 1) -def perform_evaluation_at(cur_data, cur_lambd: float, cur_massl: bool, cur_min_lv: int, cur_one_vs_others: bool): +def pea_classification(cur_lambd: float, cur_massl: bool, cur_min_lv: int, cur_one_vs_others: bool): + dataset_blobs = deml.datasets.make_blobs(n_samples=samples, n_features=dimension, centers=labels) + data_blobs = deml.DataSet(dataset_blobs, name='Blobs_Set') + cur_data = data_blobs.copy() + #should implement a smoother way to put in the data set classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) classification.perform_classification(masslumping=cur_massl, lambd=cur_lambd, minimum_level=cur_min_lv, maximum_level=5, one_vs_others=cur_one_vs_others, print_metrics=False) evaluation = classification.evaluate() @@ -340,5 +345,5 @@ def perform_evaluation_at(cur_data, cur_lambd: float, cur_massl: bool, cur_min_l #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion return evaluation["Percentage correct"] classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] -HPO = HP_Optimization(data_moons, perform_evaluation_at, classification_space) +HPO = HP_Optimization(pea_classification, classification_space) HPO.perform_evaluation_at(data_moons, 0.00242204, 0, 1, 0) From bcb8e0825ecdd60ad07a534b03378edf96997428 Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 10 Aug 2021 09:47:08 +0200 Subject: [PATCH 25/70] "data" is now only used in the function to be optimized --- sparseSpACE/HP_Optimization.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index b10052b..5204329 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -15,7 +15,7 @@ def __init__(self, function, hp_space): self.hp_space = hp_space #performs Grid Optimization - def perform_GO_classification(self, data, lambd_levels): + def perform_GO_classification(self, lambd_levels): #sklearn_dataset = deml.datasets.make_moons(n_samples=samples, noise=0.15) #data = deml.DataSet(sklearn_dataset, name='Input_Set') #for storing the current best evaluation and time @@ -41,7 +41,7 @@ def perform_GO_classification(self, data, lambd_levels): #use "perform evaluation at" #cur_time = classification._time_used #print ("current time needed = " + str(cur_time)) - cur_evaluation = perform_evaluation_at(data, cur_lambd, cur_massl, cur_min_lv, cur_one_vs_others) + cur_evaluation = perform_evaluation_at(cur_lambd, cur_massl, cur_min_lv, cur_one_vs_others) print("Percentage of correct mappings",cur_evaluation) #if best_evaluation == None or cur_evaluation > best_evaluation or (cur_evaluation == best_evaluation and (best_time == None or best_time>cur_time)): if(best_evaluation == None or cur_evaluation>best_evaluation): @@ -58,19 +58,19 @@ def perform_GO_classification(self, data, lambd_levels): #+ " and time = " + str(best_time)) #performs random optimization - def perform_RO_classification(self, data, amt_it: int, dim: int): + def perform_RO_classification(self, amt_it: int, dim: int): best_evaluation = 0 best_x = [1, 1, 1, 1] for i in range (0, amt_it, 1): x = create_random_x() - new_eval = perform_evaluation_at(data, x[0], x[1], x[2], x[3]) + new_eval = perform_evaluation_at(x[0], x[1], x[2], x[3]) if new_eval>best_evaluation: best_x = x best_evaluation = new_eval print("Best evaluation in " + str(amt_it) + " random steps: " + str(best_evaluation) + " at " + str(x)) #returns evaluation for certain Parameters on a certain data set - def perform_evaluation_at(self, cur_data, cur_lambd: float, cur_massl: bool, cur_min_lv: int, cur_one_vs_others: bool): + def perform_evaluation_at(self, cur_lambd: float, cur_massl: bool, cur_min_lv: int, cur_one_vs_others: bool): #classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) #classification.perform_classification(masslumping=cur_massl, lambd=cur_lambd, minimum_level=cur_min_lv, maximum_level=5, one_vs_others=cur_one_vs_others, print_metrics=False) #evaluation = classification.evaluate() @@ -83,14 +83,14 @@ def perform_evaluation_at(self, cur_data, cur_lambd: float, cur_massl: bool, cur #TODO: will perform Bayesian Optimization; Currently basically performs BO with Naive rounding and a weird beta #Note: HP Input of classification is (lambd:float, massl:bool, min_lv:int <4, one_vs_others:bool) - def perform_BO_classification(self, data, amt_it: int, function = perform_evaluation_at, space = classification_space, ev_is_rand: bool = True): + def perform_BO_classification(self, amt_it: int, function = perform_evaluation_at, space = classification_space, ev_is_rand: bool = True): #notes how many x values currently are in the evidence set - starts with amt_HP+1 amt_HP = len(space) cur_amt_x: int = amt_HP+1 x_ret = None y_ret = None print("I should do Bayesian Optimization for " + str(amt_it) + " iterations, ev_is_rand is " + str(ev_is_rand) + " !") - C = create_evidence_set(data, amt_it, amt_HP, ev_is_rand) + C = create_evidence_set(amt_it, amt_HP, ev_is_rand) C_x=C[0] print("C_x: " + str(C_x)) C_y=C[1] @@ -135,7 +135,7 @@ def perform_BO_classification(self, data, amt_it: int, function = perform_evalua break print("adding " + str(new_x_rd)) C_x[cur_amt_x] = new_x_rd - C_y[cur_amt_x] = perform_evaluation_at(data, new_x_rd[0], new_x_rd[1], new_x_rd[2], new_x_rd[3]) + C_y[cur_amt_x] = perform_evaluation_at(new_x_rd[0], new_x_rd[1], new_x_rd[2], new_x_rd[3]) cur_amt_x += 1 #ends everything if cov_matr is singular. Should this be dependant on l? Cov_matr being singular should not depend on l I think print("Checking if cov_matr is singular") @@ -189,7 +189,7 @@ def get_cov_vector(self, ev_x, new_x, cur_length: int): k_vec[i] = k(ev_x[i], new_x) return k_vec - def create_evidence_set(self, data, amt_it: int, dim_HP: int, is_random: bool = True): + def create_evidence_set(self, amt_it: int, dim_HP: int, is_random: bool = True): x = np.zeros(shape=(amt_it+dim_HP+1, dim_HP)) y = np.zeros((amt_it+dim_HP+1)) if(is_random): @@ -201,7 +201,7 @@ def create_evidence_set(self, data, amt_it: int, dim_HP: int, is_random: bool = #evaluating takes quite some time, especially for min_lv>1 #y[i] = perform_evaluation_at(data, new_x[0], new_x[1], new_x[2], new_x[3]) #needs to be casted to int bc otherwise it's automatically float which doesn't work - y[i] = perform_evaluation_at(data, x[i][0], int(x[i][1]), int(x[i][2]), int(x[i][3])) + y[i] = perform_evaluation_at(x[i][0], int(x[i][1]), int(x[i][2]), int(x[i][3])) else: #hard-code non-random values for testing - current threw error at it. 24 #use seed for random values instead of hard coded values? -> get used seed @@ -211,10 +211,10 @@ def create_evidence_set(self, data, amt_it: int, dim_HP: int, is_random: bool = x[3] = [0.88249225, 1., 1., 0.] #[0.51043662, 1, 1, 0] x[4] = [0.1321559, 1., 2., 1.] #[0.54776247, 0, 1, 0] for i in range(0, dim_HP+1, 1): - y[i] = perform_evaluation_at(data, x[i][0], int(x[i][1]), int(x[i][2]), int(x[i][3])) + y[i] = perform_evaluation_at(x[i][0], int(x[i][1]), int(x[i][2]), int(x[i][3])) return x, y - #returns a random x for the given purpose + #returns a random x for the given purpose - needs search space def create_random_x(self, purpose = "classification"): if(purpose == "classification"): return [random.random(), random.randint(0, 1), random.randint(1, 3), random.randint(0, 1)] @@ -298,6 +298,7 @@ def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): #print("new x: " + str(new_x) + " with function value: " + str(alpha(new_x))) return new_x + #needs search space def round_x_classification(self, x): if len(x) < 4: print("Input too short! Returning default values") @@ -346,4 +347,4 @@ def pea_classification(cur_lambd: float, cur_massl: bool, cur_min_lv: int, cur_o return evaluation["Percentage correct"] classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] HPO = HP_Optimization(pea_classification, classification_space) -HPO.perform_evaluation_at(data_moons, 0.00242204, 0, 1, 0) +HPO.perform_evaluation_at(0.00242204, 0, 1, 0) From 3c2e09fad4c463f588bcf9fbaf4e783af1044375 Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 10 Aug 2021 18:55:16 +0200 Subject: [PATCH 26/70] Push for easier copy-pasting --- sparseSpACE/HP_Optimization.py | 70 +++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 23 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 5204329..34f985f 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -41,7 +41,7 @@ def perform_GO_classification(self, lambd_levels): #use "perform evaluation at" #cur_time = classification._time_used #print ("current time needed = " + str(cur_time)) - cur_evaluation = perform_evaluation_at(cur_lambd, cur_massl, cur_min_lv, cur_one_vs_others) + cur_evaluation = self.perform_evaluation_at(cur_lambd, cur_massl, cur_min_lv, cur_one_vs_others) print("Percentage of correct mappings",cur_evaluation) #if best_evaluation == None or cur_evaluation > best_evaluation or (cur_evaluation == best_evaluation and (best_time == None or best_time>cur_time)): if(best_evaluation == None or cur_evaluation>best_evaluation): @@ -63,29 +63,29 @@ def perform_RO_classification(self, amt_it: int, dim: int): best_x = [1, 1, 1, 1] for i in range (0, amt_it, 1): x = create_random_x() - new_eval = perform_evaluation_at(x[0], x[1], x[2], x[3]) + new_eval = perform_evaluation_at(x) if new_eval>best_evaluation: best_x = x best_evaluation = new_eval print("Best evaluation in " + str(amt_it) + " random steps: " + str(best_evaluation) + " at " + str(x)) #returns evaluation for certain Parameters on a certain data set - def perform_evaluation_at(self, cur_lambd: float, cur_massl: bool, cur_min_lv: int, cur_one_vs_others: bool): + def perform_evaluation_at(self, params): #classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) #classification.perform_classification(masslumping=cur_massl, lambd=cur_lambd, minimum_level=cur_min_lv, maximum_level=5, one_vs_others=cur_one_vs_others, print_metrics=False) #evaluation = classification.evaluate() #print("Percentage of correct mappings",evaluation["Percentage correct"]) ##wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion #return evaluation["Percentage correct"] - return self.function(cur_lambd, cur_massl, cur_min_lv, cur_one_vs_others) + return self.function(params) - classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] + #classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] #TODO: will perform Bayesian Optimization; Currently basically performs BO with Naive rounding and a weird beta #Note: HP Input of classification is (lambd:float, massl:bool, min_lv:int <4, one_vs_others:bool) - def perform_BO_classification(self, amt_it: int, function = perform_evaluation_at, space = classification_space, ev_is_rand: bool = True): + def perform_BO_classification(self, amt_it: int, ev_is_rand: bool = True): #notes how many x values currently are in the evidence set - starts with amt_HP+1 - amt_HP = len(space) + amt_HP = len(self.hp_space) cur_amt_x: int = amt_HP+1 x_ret = None y_ret = None @@ -201,7 +201,7 @@ def create_evidence_set(self, amt_it: int, dim_HP: int, is_random: bool = True): #evaluating takes quite some time, especially for min_lv>1 #y[i] = perform_evaluation_at(data, new_x[0], new_x[1], new_x[2], new_x[3]) #needs to be casted to int bc otherwise it's automatically float which doesn't work - y[i] = perform_evaluation_at(x[i][0], int(x[i][1]), int(x[i][2]), int(x[i][3])) + y[i] = perform_evaluation_at(x[i]) else: #hard-code non-random values for testing - current threw error at it. 24 #use seed for random values instead of hard coded values? -> get used seed @@ -211,15 +211,26 @@ def create_evidence_set(self, amt_it: int, dim_HP: int, is_random: bool = True): x[3] = [0.88249225, 1., 1., 0.] #[0.51043662, 1, 1, 0] x[4] = [0.1321559, 1., 2., 1.] #[0.54776247, 0, 1, 0] for i in range(0, dim_HP+1, 1): - y[i] = perform_evaluation_at(x[i][0], int(x[i][1]), int(x[i][2]), int(x[i][3])) + y[i] = perform_evaluation_at(x[i]) return x, y #returns a random x for the given purpose - needs search space - def create_random_x(self, purpose = "classification"): - if(purpose == "classification"): - return [random.random(), random.randint(0, 1), random.randint(1, 3), random.randint(0, 1)] - else: - print("Invalid Input") + def create_random_x(self): + res = [] + for i in range (0, len(self.hp_space)): + new_x = 1 + if (len(self.hp_space[i])<3): + print("Too little arguments in HP Space! Using default value 1 for index " + str(i)) + elif (self.hp_space[i][0] == "interval"): + new_x = random.uniform(self.hp_space[i][1], self.hp_space[i][2]) + elif (self.hp_space[i][0] == "list"): + sel = self.hp_space[i] + sel.remove("list") + new_x = random.choice(sel) + else: + print("Unknown type of space! Using default value 1 for index " + str(i)) + res.append(new_x) + return res #checks if array has the element x, returns True if yes. Caution: If x has same length but different dimension to the elements of array this may give unintended results #maybe later a threshhold can be added if only part of the matrix should be checked @@ -299,11 +310,14 @@ def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): return new_x #needs search space - def round_x_classification(self, x): - if len(x) < 4: + def round_x(self, x): + if len(x) < len(self.hp_space): print("Input too short! Returning default values") - return [0, 0, 1, 0] - if len(x) > 4: + default = np.zeros(len(self.hp_space)) + if(len(default)>2): + default[2] = 1 + return default + if len(x) > len(self.hp_space): print("Input too long! Cropping") #rounds the values of new_x to values usable as HPs - the question is what kind of rounding makes sense #e.g if lambda should be bounded or more values should be rounded to 0 @@ -313,7 +327,10 @@ def round_x_classification(self, x): if(min_lv<1): min_lv = 1 elif (min_lv>3): min_lv = 3 one_vs_others = math.trunc(x[3]) - new_x_rd = [lambd, massl, min_lv, one_vs_others] + new_x_rd = [] + for i in range (0, len(x)): + "EEEEEEK" + return new_x_rd #Basically copied code from Tutorial_DEMachineLearning @@ -334,17 +351,24 @@ def round_x_classification(self, x): #perform_RO_classification(data_moons, 10, dimension) #perform_GO_classification(data_moons, 20) #perform_evaluation_at(self, data_moons, 0.828603059876013, 0, 2, 1) -def pea_classification(cur_lambd: float, cur_massl: bool, cur_min_lv: int, cur_one_vs_others: bool): - dataset_blobs = deml.datasets.make_blobs(n_samples=samples, n_features=dimension, centers=labels) +#parameters are in form lambd, masslump, minlv, one_vs_others +def pea_classification(params): + dataset_blobs = deml.datasets.make_blobs(n_samples=samples, n_features=dimension, centers=labels, random_state=1) data_blobs = deml.DataSet(dataset_blobs, name='Blobs_Set') cur_data = data_blobs.copy() + #dataset_moons = deml.datasets.make_moons(n_samples=samples, noise=0.15, random_state=1) + #data_moons = deml.DataSet(sklearn_dataset, name='data_moons') + #cur_data = data_moons.copy() #should implement a smoother way to put in the data set classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) - classification.perform_classification(masslumping=cur_massl, lambd=cur_lambd, minimum_level=cur_min_lv, maximum_level=5, one_vs_others=cur_one_vs_others, print_metrics=False) + classification.perform_classification(masslumping=params[1], lambd=params[0], minimum_level=params[2], maximum_level=5, one_vs_others=params[3], print_metrics=False) evaluation = classification.evaluate() print("Percentage of correct mappings",evaluation["Percentage correct"]) #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion return evaluation["Percentage correct"] + classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] HPO = HP_Optimization(pea_classification, classification_space) -HPO.perform_evaluation_at(0.00242204, 0, 1, 0) +#HPO.perform_evaluation_at([0.00242204, 0, 1, 0]) +#HPO.perform_BO_classification(5) +print(HPO.create_random_x()) From dde80dcee5ef9e739ff1f1b85b91e23ddd76de8d Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 10 Aug 2021 19:32:07 +0200 Subject: [PATCH 27/70] end of the day push --- sparseSpACE/HP_Optimization.py | 51 ++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 34f985f..07986b2 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -62,8 +62,8 @@ def perform_RO_classification(self, amt_it: int, dim: int): best_evaluation = 0 best_x = [1, 1, 1, 1] for i in range (0, amt_it, 1): - x = create_random_x() - new_eval = perform_evaluation_at(x) + x = self.create_random_x() + new_eval = self.perform_evaluation_at(x) if new_eval>best_evaluation: best_x = x best_evaluation = new_eval @@ -90,7 +90,7 @@ def perform_BO_classification(self, amt_it: int, ev_is_rand: bool = True): x_ret = None y_ret = None print("I should do Bayesian Optimization for " + str(amt_it) + " iterations, ev_is_rand is " + str(ev_is_rand) + " !") - C = create_evidence_set(amt_it, amt_HP, ev_is_rand) + C = self.create_evidence_set(amt_it, amt_HP, ev_is_rand) C_x=C[0] print("C_x: " + str(C_x)) C_y=C[1] @@ -98,12 +98,12 @@ def perform_BO_classification(self, amt_it: int, ev_is_rand: bool = True): #print("Evidence Set: \n" + str(C)) for i in range (0, amt_it, 1): print("iteration: " + str(i)) - beta = get_beta(i+1) - l = get_l_k(i) - if(np.linalg.det(get_cov_matrix(C_x, cur_amt_x))==0): + beta = self.get_beta(i+1) + l = self.get_l_k(i) + if(np.linalg.det(self.get_cov_matrix(C_x, cur_amt_x))==0): print("With the nex x value added the Covariance Matrix is singular.\nBayesian Optimization can not be continued and will be ended here, at the start of iteration " + str(i)) break - if((get_cov_matrix(C_x, cur_amt_x) == np.identity(len(C_x))*(cur_amt_x+10)).all()): + if((self.get_cov_matrix(C_x, cur_amt_x) == np.identity(len(C_x))*(cur_amt_x+10)).all()): print("cov_matr has been modified to be Id*scalar! This means it was singular -> ending at iteration " + str(i)) break else: @@ -111,18 +111,18 @@ def perform_BO_classification(self, amt_it: int, ev_is_rand: bool = True): #value that will be evaluated and added to the evidence set print("Getting new x:") - new_x=acq_x(beta, l, C_x, C_y, cur_amt_x) + new_x=self.acq_x(beta, l, C_x, C_y, cur_amt_x) print("new x: " + str(new_x)) - new_x_rd = round_x_classification(new_x) + new_x_rd = self.round_x(new_x) print("new x rd: " + str(new_x_rd)) #erstelle neues beta und l und berechne neuen wert damit while(check_if_in_array(new_x_rd, C_x)): print("!!!!!!!!!!!Need new beta and l") old_x_rd = new_x_rd print("old x rd was: " + str(old_x_rd)) - beta_and_l = get_new_beta_and_l(beta, cur_amt_x, new_x, C_x, C_y) - new_x = acq_x(beta_and_l[0], beta_and_l[1], C_x, C_y, cur_amt_x) - new_x_rd = round_x_classification(new_x) + beta_and_l = self.get_new_beta_and_l(beta, cur_amt_x, new_x, C_x, C_y) + new_x = self.acq_x(beta_and_l[0], beta_and_l[1], C_x, C_y, cur_amt_x) + new_x_rd = self.round_x(new_x) print("new x rd is: " + str(new_x_rd)) if(old_x_rd==new_x_rd): print("We're in an infinite loop! Getting out") @@ -135,14 +135,14 @@ def perform_BO_classification(self, amt_it: int, ev_is_rand: bool = True): break print("adding " + str(new_x_rd)) C_x[cur_amt_x] = new_x_rd - C_y[cur_amt_x] = perform_evaluation_at(new_x_rd[0], new_x_rd[1], new_x_rd[2], new_x_rd[3]) + C_y[cur_amt_x] = self.perform_evaluation_at(new_x_rd) cur_amt_x += 1 #ends everything if cov_matr is singular. Should this be dependant on l? Cov_matr being singular should not depend on l I think print("Checking if cov_matr is singular") - if(np.linalg.det(get_cov_matrix(C_x, cur_amt_x)) == 0): + if(np.linalg.det(self.get_cov_matrix(C_x, cur_amt_x)) == 0): print("With the nex x value added the Covariance Matrix is singular.\nBayesian Optimization can not be continued and will be ended here, after " + str(i) + " iterations") break - if((get_cov_matrix(C_x, cur_amt_x) == np.identity(len(C_x))*(cur_amt_x+10)).all()): + if((self.get_cov_matrix(C_x, cur_amt_x) == np.identity(len(C_x))*(cur_amt_x+10)).all()): print("cov_matr has been modified to be Id*scalar! This means it was singular -> ending at iteration " + str(i)) break else: @@ -158,8 +158,9 @@ def perform_BO_classification(self, amt_it: int, ev_is_rand: bool = True): sigma_k = 1 l_k = 0.5 #since creation of covariance matrices is done in another function k(x) needs to be outside - k = lambda x, y: (sigma_k**2)*math.exp((-0.5/l_k**2)*np.linalg.norm(x-y)**2) + k = lambda x, y: (self.sigma_k**2)*math.exp((-0.5/self.l_k**2)*np.linalg.norm(x-y)**2) + "BIS HIER self. eingefügt (hoffentlich)" #cur_length is needed so that rest of matrix stays Id def get_cov_matrix(self, C_x, cur_length: int): K = np.identity(len(C_x)) @@ -329,8 +330,22 @@ def round_x(self, x): one_vs_others = math.trunc(x[3]) new_x_rd = [] for i in range (0, len(x)): - "EEEEEEK" - + new_x = 1 + if(len(self.hp_space[i])<3): + print("Too little arguments in HP Space! Using default value 1 for index " + str(i)) + elif(self.hp_space[i][0] == "interval"): + if(x[i]self.hp_space[i][2]): + new_x = self.hp_space[i][2] + else: + new_x = x[i] + elif(self.hp_space[i][0] == "list"): + new_x = 1 + #maybe work with how far away from possible values x[i] is? + else: + print("Unknown type of space! Using default value 1 for index " + str(i)) + new_x_rd.append(new_x) return new_x_rd #Basically copied code from Tutorial_DEMachineLearning From 7b81b29027d23bc0ae94b2453d2154e67265a39a Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 10 Aug 2021 19:50:09 +0200 Subject: [PATCH 28/70] Added "self" everywhere necessary (I hope) --- sparseSpACE/HP_Optimization.py | 35 +++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 07986b2..8c6f3f1 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -116,7 +116,7 @@ def perform_BO_classification(self, amt_it: int, ev_is_rand: bool = True): new_x_rd = self.round_x(new_x) print("new x rd: " + str(new_x_rd)) #erstelle neues beta und l und berechne neuen wert damit - while(check_if_in_array(new_x_rd, C_x)): + while(self.check_if_in_array(new_x_rd, C_x)): print("!!!!!!!!!!!Need new beta and l") old_x_rd = new_x_rd print("old x rd was: " + str(old_x_rd)) @@ -158,15 +158,16 @@ def perform_BO_classification(self, amt_it: int, ev_is_rand: bool = True): sigma_k = 1 l_k = 0.5 #since creation of covariance matrices is done in another function k(x) needs to be outside - k = lambda x, y: (self.sigma_k**2)*math.exp((-0.5/self.l_k**2)*np.linalg.norm(x-y)**2) + def cov(self, x, y): + k = lambda x, y: (self.sigma_k**2)*math.exp((-0.5/self.l_k**2)*np.linalg.norm(x-y)**2) + return k(x, y) - "BIS HIER self. eingefügt (hoffentlich)" #cur_length is needed so that rest of matrix stays Id def get_cov_matrix(self, C_x, cur_length: int): K = np.identity(len(C_x)) for i in range (0, cur_length, 1): for j in range (0, cur_length, 1): - K[i][j]=k(C_x[i], C_x[j]) + K[i][j]=self.cov(C_x[i], C_x[j]) if(np.linalg.det(K) == 0): print("Oh no! The covariance matrix is singular!! It is " + str(K)) print("Current C_x is: " + str(C_x)) @@ -187,7 +188,7 @@ def get_cov_vector(self, ev_x, new_x, cur_length: int): return None k_vec = np.zeros(len(ev_x)) for i in range (0, cur_length, 1): - k_vec[i] = k(ev_x[i], new_x) + k_vec[i] = cov(ev_x[i], new_x) return k_vec def create_evidence_set(self, amt_it: int, dim_HP: int, is_random: bool = True): @@ -195,14 +196,14 @@ def create_evidence_set(self, amt_it: int, dim_HP: int, is_random: bool = True): y = np.zeros((amt_it+dim_HP+1)) if(is_random): for i in range(0, dim_HP+1, 1): - new_x = create_random_x() - while(check_if_in_array(new_x, x)): - new_x = create_random_x() + new_x = self.create_random_x() + while(self.check_if_in_array(new_x, x)): + new_x = self.create_random_x() x[i] = new_x #evaluating takes quite some time, especially for min_lv>1 #y[i] = perform_evaluation_at(data, new_x[0], new_x[1], new_x[2], new_x[3]) #needs to be casted to int bc otherwise it's automatically float which doesn't work - y[i] = perform_evaluation_at(x[i]) + y[i] = self.perform_evaluation_at(x[i]) else: #hard-code non-random values for testing - current threw error at it. 24 #use seed for random values instead of hard coded values? -> get used seed @@ -212,7 +213,7 @@ def create_evidence_set(self, amt_it: int, dim_HP: int, is_random: bool = True): x[3] = [0.88249225, 1., 1., 0.] #[0.51043662, 1, 1, 0] x[4] = [0.1321559, 1., 2., 1.] #[0.54776247, 0, 1, 0] for i in range(0, dim_HP+1, 1): - y[i] = perform_evaluation_at(x[i]) + y[i] = self.perform_evaluation_at(x[i]) return x, y #returns a random x for the given purpose - needs search space @@ -264,7 +265,7 @@ def get_l_k(self, t: int): def get_new_beta_and_l(self, cur_beta: float, cur_amt_x, cur_x, C_x, C_y): global l_k print("Getting new beta and l. Current beta: " + str(cur_beta) +", Current l: " + str(l_k)) - if(np.linalg.det(get_cov_matrix(C_x, cur_amt_x))==0): + if(np.linalg.det(self.get_cov_matrix(C_x, cur_amt_x))==0): print("Getting new beta and l, but suddenly the cov matr is singular??") else: print("Apparently the cov_matr is not singular. Getting new beta and l") @@ -273,9 +274,9 @@ def get_new_beta_and_l(self, cur_beta: float, cur_amt_x, cur_x, C_x, C_y): l_h = l_k+50 #0 if rd(x) is in ev set C_x, constant otherwise (5) #penalty p(x) erhöhen wenn gleicher Wert rauskommt. z.B. immer +50 oder *2 oder anders exponentiell - p = lambda x: check_if_in_array(round_x_classification(x), C_x)*50 + p = lambda x: self.check_if_in_array(round_x_classification(x), C_x)*50 #gets the x value for certain l (z[1]) and beta (z[0]) - new_x = lambda z: acq_x(z[0], z[1], C_x, C_y, cur_amt_x) + new_x = lambda z: self.acq_x(z[0], z[1], C_x, C_y, cur_amt_x) #for g: x[0] is \beta+d\beta, x[1] is l. Also d\beta = \beta+d\beta-\beta g = lambda x: (x[0]-cur_beta)+np.linalg.norm(cur_x-new_x(x))+p(new_x(x)) bounds_g = ((cur_beta, beta_h), (l_k, l_h)) @@ -292,15 +293,15 @@ def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): global l_k old_l=l_k l_k=l - K_matr = get_cov_matrix(C_x, cur_amt_x) + K_matr = self.get_cov_matrix(C_x, cur_amt_x) if(np.linalg.det(K_matr) == 0): print("Covariance Matrix is indeed singular!") #return a value that's bullshit? like [0, 0, 0, 0] (min lv is >=1) #print(K_matr) - mu = lambda x: get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(C_y)) - sigma_sqrd = lambda x: k(x, x)-get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(get_cov_vector(C_x, x, cur_amt_x))) + mu = lambda x: self.get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(C_y)) + sigma_sqrd = lambda x: self.cov(x, x)-self.get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(self.get_cov_vector(C_x, x, cur_amt_x))) #takes sqrt of abs(sigma_sqrd) bc otherwise fmin gives an error - might be an overall warning sign tho - sigma = lambda x: math.sqrt(abs(sigma_sqrd(x))) + sigma = lambda x: math.sqrt(abs(self.sigma_sqrd(x))) alpha = lambda x: mu(x)+(math.sqrt(beta))*sigma(x) #negates alpha bc maximum has to be found alpha_neg = lambda x: -alpha(x) From 7696e97568b9cb41298768a7b283d3a70366f33a Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 10 Aug 2021 19:54:18 +0200 Subject: [PATCH 29/70] Same as the last --- sparseSpACE/HP_Optimization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 8c6f3f1..11d623f 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -188,7 +188,7 @@ def get_cov_vector(self, ev_x, new_x, cur_length: int): return None k_vec = np.zeros(len(ev_x)) for i in range (0, cur_length, 1): - k_vec[i] = cov(ev_x[i], new_x) + k_vec[i] = self.cov(ev_x[i], new_x) return k_vec def create_evidence_set(self, amt_it: int, dim_HP: int, is_random: bool = True): @@ -274,7 +274,7 @@ def get_new_beta_and_l(self, cur_beta: float, cur_amt_x, cur_x, C_x, C_y): l_h = l_k+50 #0 if rd(x) is in ev set C_x, constant otherwise (5) #penalty p(x) erhöhen wenn gleicher Wert rauskommt. z.B. immer +50 oder *2 oder anders exponentiell - p = lambda x: self.check_if_in_array(round_x_classification(x), C_x)*50 + p = lambda x: self.check_if_in_array(self.round_x(x), C_x)*50 #gets the x value for certain l (z[1]) and beta (z[0]) new_x = lambda z: self.acq_x(z[0], z[1], C_x, C_y, cur_amt_x) #for g: x[0] is \beta+d\beta, x[1] is l. Also d\beta = \beta+d\beta-\beta From 6ef805cb42b0384ec3e1d01f85494952b238b106 Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 10 Aug 2021 20:15:28 +0200 Subject: [PATCH 30/70] same as the last --- sparseSpACE/HP_Optimization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 11d623f..1fb2174 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -67,7 +67,7 @@ def perform_RO_classification(self, amt_it: int, dim: int): if new_eval>best_evaluation: best_x = x best_evaluation = new_eval - print("Best evaluation in " + str(amt_it) + " random steps: " + str(best_evaluation) + " at " + str(x)) + print("Best evaluation in " + str(amt_it) + " random steps: " + str(best_evaluation) + " at " + str(best_x)) #returns evaluation for certain Parameters on a certain data set def perform_evaluation_at(self, params): @@ -301,7 +301,7 @@ def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): mu = lambda x: self.get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(C_y)) sigma_sqrd = lambda x: self.cov(x, x)-self.get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(self.get_cov_vector(C_x, x, cur_amt_x))) #takes sqrt of abs(sigma_sqrd) bc otherwise fmin gives an error - might be an overall warning sign tho - sigma = lambda x: math.sqrt(abs(self.sigma_sqrd(x))) + sigma = lambda x: math.sqrt(abs(sigma_sqrd(x))) alpha = lambda x: mu(x)+(math.sqrt(beta))*sigma(x) #negates alpha bc maximum has to be found alpha_neg = lambda x: -alpha(x) From 2815481df3874321b5583a9ff0ba0a93786fba69 Mon Sep 17 00:00:00 2001 From: Sonja Date: Wed, 11 Aug 2021 10:52:12 +0200 Subject: [PATCH 31/70] round_x() works with given hp_space now --- sparseSpACE/HP_Optimization.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 1fb2174..c713cd0 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -314,11 +314,9 @@ def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): #needs search space def round_x(self, x): if len(x) < len(self.hp_space): - print("Input too short! Returning default values") - default = np.zeros(len(self.hp_space)) - if(len(default)>2): - default[2] = 1 - return default + print("Input too short! Returning default values rd(0) for missing values") + for k in range (len(x), len(self.hp_space)): + x.append(0) if len(x) > len(self.hp_space): print("Input too long! Cropping") #rounds the values of new_x to values usable as HPs - the question is what kind of rounding makes sense @@ -330,7 +328,7 @@ def round_x(self, x): elif (min_lv>3): min_lv = 3 one_vs_others = math.trunc(x[3]) new_x_rd = [] - for i in range (0, len(x)): + for i in range (0, len(self.hp_space)): new_x = 1 if(len(self.hp_space[i])<3): print("Too little arguments in HP Space! Using default value 1 for index " + str(i)) @@ -342,8 +340,14 @@ def round_x(self, x): else: new_x = x[i] elif(self.hp_space[i][0] == "list"): - new_x = 1 - #maybe work with how far away from possible values x[i] is? + new_x = self.hp_space[i][1] + dis = abs(x[i]-self.hp_space[i][1]) + #takes the value in the list that is least away from x + for j in range (2, len(self.hp_space[i])): + cur_dis = abs(x[i]-self.hp_space[i][j]) + if(cur_dis < dis): + dis = cur_dis + new_x = self.hp_space[i][j] else: print("Unknown type of space! Using default value 1 for index " + str(i)) new_x_rd.append(new_x) @@ -383,8 +387,10 @@ def pea_classification(params): #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion return evaluation["Percentage correct"] -classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] +classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1], ["interval", 3, 4, 5], ["blues", 4, 5, 6]] HPO = HP_Optimization(pea_classification, classification_space) #HPO.perform_evaluation_at([0.00242204, 0, 1, 0]) #HPO.perform_BO_classification(5) -print(HPO.create_random_x()) +print(HPO.round_x([0.345])) +print(HPO.round_x([-3.75, -1000, -3, 1.567890234, 3.757575, 3000000, 4, 5, 4.5677])) +print(HPO.round_x([-3.75, -1000, -3, 1.567890234, 3.757575, 5])) From 705be8f918d22ee7d7a34aa99dfd97599f3093c9 Mon Sep 17 00:00:00 2001 From: Sonja Date: Wed, 11 Aug 2021 13:19:46 +0200 Subject: [PATCH 32/70] perform_BO works with given function and space - at least for classification --- sparseSpACE/HP_Optimization.py | 60 ++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index c713cd0..3216614 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -77,7 +77,10 @@ def perform_evaluation_at(self, params): #print("Percentage of correct mappings",evaluation["Percentage correct"]) ##wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion #return evaluation["Percentage correct"] - return self.function(params) + print("performing evaluation at " + str(params)) + res = self.function(params) + print("returning " + str(res)) + return res #classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] @@ -114,11 +117,12 @@ def perform_BO_classification(self, amt_it: int, ev_is_rand: bool = True): new_x=self.acq_x(beta, l, C_x, C_y, cur_amt_x) print("new x: " + str(new_x)) new_x_rd = self.round_x(new_x) + old_x_rd = None print("new x rd: " + str(new_x_rd)) #erstelle neues beta und l und berechne neuen wert damit while(self.check_if_in_array(new_x_rd, C_x)): print("!!!!!!!!!!!Need new beta and l") - old_x_rd = new_x_rd + old_x_rd = new_x_rd.copy() print("old x rd was: " + str(old_x_rd)) beta_and_l = self.get_new_beta_and_l(beta, cur_amt_x, new_x, C_x, C_y) new_x = self.acq_x(beta_and_l[0], beta_and_l[1], C_x, C_y, cur_amt_x) @@ -191,7 +195,8 @@ def get_cov_vector(self, ev_x, new_x, cur_length: int): k_vec[i] = self.cov(ev_x[i], new_x) return k_vec - def create_evidence_set(self, amt_it: int, dim_HP: int, is_random: bool = True): +#maybe give method the non-rand values instead of hardcoding them? + def create_evidence_set(self, amt_it: int, dim_HP: int, is_random: bool = True, non_rand_values = None): x = np.zeros(shape=(amt_it+dim_HP+1, dim_HP)) y = np.zeros((amt_it+dim_HP+1)) if(is_random): @@ -218,17 +223,19 @@ def create_evidence_set(self, amt_it: int, dim_HP: int, is_random: bool = True): #returns a random x for the given purpose - needs search space def create_random_x(self): + print("creating random x") res = [] for i in range (0, len(self.hp_space)): new_x = 1 if (len(self.hp_space[i])<3): - print("Too little arguments in HP Space! Using default value 1 for index " + str(i)) + print("Too little Arguments! HP Space index " + str(i) + ": " + str(self.hp_space[i])) elif (self.hp_space[i][0] == "interval"): new_x = random.uniform(self.hp_space[i][1], self.hp_space[i][2]) elif (self.hp_space[i][0] == "list"): - sel = self.hp_space[i] + sel = self.hp_space[i].copy() sel.remove("list") new_x = random.choice(sel) + print("hp_space[" + str(i) + "] is now " + str(self.hp_space[i])) else: print("Unknown type of space! Using default value 1 for index " + str(i)) res.append(new_x) @@ -263,15 +270,14 @@ def get_l_k(self, t: int): #TODO: get new beta and l. Currently mock function def get_new_beta_and_l(self, cur_beta: float, cur_amt_x, cur_x, C_x, C_y): - global l_k - print("Getting new beta and l. Current beta: " + str(cur_beta) +", Current l: " + str(l_k)) + print("Getting new beta and l. Current beta: " + str(cur_beta) +", Current l: " + str(self.l_k)) if(np.linalg.det(self.get_cov_matrix(C_x, cur_amt_x))==0): print("Getting new beta and l, but suddenly the cov matr is singular??") else: print("Apparently the cov_matr is not singular. Getting new beta and l") #making upper bounds dependable on current values so bounds are never too low beta_h = cur_beta+100 - l_h = l_k+50 + l_h = self.l_k+50 #0 if rd(x) is in ev set C_x, constant otherwise (5) #penalty p(x) erhöhen wenn gleicher Wert rauskommt. z.B. immer +50 oder *2 oder anders exponentiell p = lambda x: self.check_if_in_array(self.round_x(x), C_x)*50 @@ -279,7 +285,7 @@ def get_new_beta_and_l(self, cur_beta: float, cur_amt_x, cur_x, C_x, C_y): new_x = lambda z: self.acq_x(z[0], z[1], C_x, C_y, cur_amt_x) #for g: x[0] is \beta+d\beta, x[1] is l. Also d\beta = \beta+d\beta-\beta g = lambda x: (x[0]-cur_beta)+np.linalg.norm(cur_x-new_x(x))+p(new_x(x)) - bounds_g = ((cur_beta, beta_h), (l_k, l_h)) + bounds_g = ((cur_beta, beta_h), (self.l_k, l_h)) print("About to minimize g(...)") #due to numerical inaccuracies a matrix might become singular with new beta and l #(even though it wouldn't be) mathematically - how to deal with that?? @@ -290,9 +296,8 @@ def get_new_beta_and_l(self, cur_beta: float, cur_amt_x, cur_x, C_x, C_y): #TODO acquire new x for l, beta, C_x, cur_amt_x, using GP-UCB def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): - global l_k - old_l=l_k - l_k=l + old_l=self.l_k + self.l_k=l K_matr = self.get_cov_matrix(C_x, cur_amt_x) if(np.linalg.det(K_matr) == 0): print("Covariance Matrix is indeed singular!") @@ -313,6 +318,7 @@ def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): #needs search space def round_x(self, x): + print("rounding x") if len(x) < len(self.hp_space): print("Input too short! Returning default values rd(0) for missing values") for k in range (len(x), len(self.hp_space)): @@ -321,16 +327,11 @@ def round_x(self, x): print("Input too long! Cropping") #rounds the values of new_x to values usable as HPs - the question is what kind of rounding makes sense #e.g if lambda should be bounded or more values should be rounded to 0 - lambd = x[0] - massl = math.trunc(x[1]) #is true if abs val > 1 - min_lv = math.trunc(x[2]) - if(min_lv<1): min_lv = 1 - elif (min_lv>3): min_lv = 3 - one_vs_others = math.trunc(x[3]) new_x_rd = [] for i in range (0, len(self.hp_space)): new_x = 1 if(len(self.hp_space[i])<3): + print("Too little Arguments! HP Space index " + str(i) + ": " + str(self.hp_space[i])) print("Too little arguments in HP Space! Using default value 1 for index " + str(i)) elif(self.hp_space[i][0] == "interval"): if(x[i] Date: Wed, 11 Aug 2021 20:38:08 +0200 Subject: [PATCH 33/70] added a simple testfunction to optimize and fixed errors trying that threw --- sparseSpACE/HP_Optimization.py | 129 ++++++++++++++++++++++----------- 1 file changed, 87 insertions(+), 42 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 3216614..c0f8c6d 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -3,6 +3,7 @@ import scipy as sp import math import random +import itertools from scipy.optimize import fmin from scipy.optimize import minimize @@ -15,52 +16,65 @@ def __init__(self, function, hp_space): self.hp_space = hp_space #performs Grid Optimization - def perform_GO_classification(self, lambd_levels): + def perform_GO(self, int_levels): #sklearn_dataset = deml.datasets.make_moons(n_samples=samples, noise=0.15) #data = deml.DataSet(sklearn_dataset, name='Input_Set') #for storing the current best evaluation and time best_evaluation = None #best_time = None #storing the parameters of the best evaluation - best_lambd = None - best_masslump = None - best_min_lv = None - best_one_vs_others = None + best_x = None #todo: split Dataset?? #original classification: classification = deml.Classification(data, split_percentage=0.8, split_evenly=True, shuffle_data=True) #original settings: classification.perform_classification(masslumping=True, lambd=0.0, minimum_level=1, maximum_level=5, print_metrics=True) #perform grid search for lambd_levels many lambd between 0 and 1 and all values for the rest #Note: lambd vllt besser logarithmisch, sehr kleine Werte oft besser (1/10, 1/100, 1/1000) #für for-schleife karthesisches produkt aus HO um alle tupel zu bilden und über diese tupel zu iterieren für beliebig dimensional - for cur_lambd_it in range (0, lambd_levels, 1): - cur_lambd = cur_lambd_it/lambd_levels - #Note: lambd vor Allem sinnvoll wenn masslumping aus ist, sollte kaum unterschied machen wenn an - for cur_massl in [True, False]: - for cur_min_lv in range (1, 3, 1): - for cur_one_vs_others in [True, False]: - #use "perform evaluation at" - #cur_time = classification._time_used - #print ("current time needed = " + str(cur_time)) - cur_evaluation = self.perform_evaluation_at(cur_lambd, cur_massl, cur_min_lv, cur_one_vs_others) - print("Percentage of correct mappings",cur_evaluation) - #if best_evaluation == None or cur_evaluation > best_evaluation or (cur_evaluation == best_evaluation and (best_time == None or best_time>cur_time)): - if(best_evaluation == None or cur_evaluation>best_evaluation): - best_evaluation = cur_evaluation - best_lambd = cur_lambd - #best_time = cur_time0 - best_masslump = cur_massl - best_min_lv = cur_min_lv - best_one_vs_others = cur_one_vs_others - print("Best evaluation is now " + str(best_evaluation) + " with lambd = " + str(best_lambd) + ", masslump = " + str(best_masslump) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others)) #+ " and time = " + str(best_time)) - else: print("Best evaluation is still " + str(best_evaluation) + " with lambd = " + str(best_lambd) + ", masslump = " + str(best_masslump) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others)) #+ " and time = " + str(best_time)) - print ("END OF CURRENT EVALUATION \n") - print("In the end, best evaluation is " + str(best_evaluation) + " with lambd = " + str(best_lambd) + ", masslump = " + str(best_masslump) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others)) + #Note: lambd vor Allem sinnvoll wenn masslumping aus ist, sollte kaum unterschied machen wenn an + search_space = self.cart_prod_hp_space(int_levels) + for x in search_space: + #use "perform evaluation at" + #cur_time = classification._time_used + #print ("current time needed = " + str(cur_time)) + cur_evaluation = self.perform_evaluation_at(x) + print("Percentage of correct mappings",cur_evaluation) + #if best_evaluation == None or cur_evaluation > best_evaluation or (cur_evaluation == best_evaluation and (best_time == None or best_time>cur_time)): + if(best_evaluation == None or cur_evaluation>best_evaluation): + best_evaluation = cur_evaluation + best_x = x + print("Best evaluation is now " + str(best_evaluation) + " at " + str(x)) + else: print("Best evaluation is still " + str(best_evaluation) + " at " + str(x)) + print ("END OF CURRENT EVALUATION \n") + print("In the end, best evaluation is " + str(best_evaluation) + " at " + str(x)) #+ " and time = " + str(best_time)) + def cart_prod_hp_space(self, interval_levels): + res = [] + for i in range (0, len(self.hp_space)): + new = [] + if(len(self.hp_space[i])<3): + print("please enter enough values for hp_space") + new.append(1) + elif(self.hp_space[i][0] == "interval"): + h = self.hp_space[i][2]-self.hp_space[i][1] + dh = h /interval_levels + #currently adds 1 more than interval_levels so that both borders are included - is there a better way to do this? + for j in range (0, interval_levels+1): + new.append(self.hp_space[i][1]+j*dh) + elif(self.hp_space[i][0] == "list"): + new = self.hp_space[i].copy() + new.remove("list") + else: + print("please enter valid types for hp_space") + new.append[1] + res.append(new) + res_cart = list(itertools.product(*res)) + return res_cart + #performs random optimization - def perform_RO_classification(self, amt_it: int, dim: int): + def perform_RO(self, amt_it: int): best_evaluation = 0 - best_x = [1, 1, 1, 1] + best_x = None for i in range (0, amt_it, 1): x = self.create_random_x() new_eval = self.perform_evaluation_at(x) @@ -84,9 +98,9 @@ def perform_evaluation_at(self, params): #classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] - #TODO: will perform Bayesian Optimization; Currently basically performs BO with Naive rounding and a weird beta + #performs Bayesian Optimization #Note: HP Input of classification is (lambd:float, massl:bool, min_lv:int <4, one_vs_others:bool) - def perform_BO_classification(self, amt_it: int, ev_is_rand: bool = True): + def perform_BO(self, amt_it: int, ev_is_rand: bool = True): #notes how many x values currently are in the evidence set - starts with amt_HP+1 amt_HP = len(self.hp_space) cur_amt_x: int = amt_HP+1 @@ -195,7 +209,7 @@ def get_cov_vector(self, ev_x, new_x, cur_length: int): k_vec[i] = self.cov(ev_x[i], new_x) return k_vec -#maybe give method the non-rand values instead of hardcoding them? + #maybe give method the non-rand values instead of hardcoding them? def create_evidence_set(self, amt_it: int, dim_HP: int, is_random: bool = True, non_rand_values = None): x = np.zeros(shape=(amt_it+dim_HP+1, dim_HP)) y = np.zeros((amt_it+dim_HP+1)) @@ -268,7 +282,7 @@ def get_beta(self, t: int): def get_l_k(self, t: int): return 0.5 - #TODO: get new beta and l. Currently mock function + #gets new beta and l if old beta and l give values that are already in C def get_new_beta_and_l(self, cur_beta: float, cur_amt_x, cur_x, C_x, C_y): print("Getting new beta and l. Current beta: " + str(cur_beta) +", Current l: " + str(self.l_k)) if(np.linalg.det(self.get_cov_matrix(C_x, cur_amt_x))==0): @@ -294,7 +308,32 @@ def get_new_beta_and_l(self, cur_beta: float, cur_amt_x, cur_x, C_x, C_y): #result is in the form [new beta, new l] return result - #TODO acquire new x for l, beta, C_x, cur_amt_x, using GP-UCB + def get_bounds(self): + res = [] + for i in range(0, len(self.hp_space)): + new = [0, 1] + #should one be able to enter just one value for "list"?? + if(len(self.hp_space[i])<3): + print("please enter a valid hp_space") + elif(self.hp_space[i][0] == "interval"): + new = self.hp_space[i].copy() + new.remove("interval") + new = [new[0], new[1]] + elif(self.hp_space[i][0] == "list"): + max = self.hp_space[i][1] + min = max + for j in range (2, len(self.hp_space[i])): + if(self.hp_space[i][j]max): + max = self.hp_space[i][j] + new = [min, max] + else: + print("please enter a valid hp_space") + res.append(new) + return res + + #acquire new x for l, beta, C_x, cur_amt_x, using GP-UCB def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): old_l=self.l_k self.l_k=l @@ -310,9 +349,10 @@ def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): alpha = lambda x: mu(x)+(math.sqrt(beta))*sigma(x) #negates alpha bc maximum has to be found alpha_neg = lambda x: -alpha(x) - bounds_an = ((0, 3), (0, 1), (1, 3), (0, 1)) #bounds search space to vicinity of useful values + bounds_an = self.get_bounds() #bounds search space to vicinity of useful values #problem: Nelder-Mead (same as fmin) cannot handle bounds -> use L-BFGS-B for now - new_x=minimize(alpha_neg, [0, 0, 0, 0], method='L-BFGS-B', bounds=bounds_an).x + x0 = np.zeros(len(self.hp_space)) + new_x=minimize(alpha_neg, x0, method='L-BFGS-B', bounds=bounds_an).x #print("new x: " + str(new_x) + " with function value: " + str(alpha(new_x))) return new_x @@ -368,7 +408,6 @@ def round_x(self, x): #perform_evaluation_at(data_moons, 0.00242204, 0, 1, 0) #.98 evaluation! #perform_BO_classification(data_blobs, 20) #perform_RO_classification(data_moons, 10, dimension) -#perform_GO_classification(data_moons, 20) #perform_evaluation_at(self, data_moons, 0.828603059876013, 0, 2, 1) #parameters are in form lambd, masslump, minlv, one_vs_others def pea_classification(params): @@ -381,7 +420,7 @@ def pea_classification(params): #cur_data = data_blobs.copy() #dataset_moons = deml.datasets.make_moons(n_samples=samples, noise=0.15, random_state=1) #data_moons = deml.DataSet(sklearn_dataset, name='data_moons') - cur_data = data_moons.copy() + cur_data = data.copy() #should implement a smoother way to put in the data set classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) classification.perform_classification(masslumping=params[1], lambd=params[0], minimum_level=params[2], maximum_level=5, one_vs_others=params[3], print_metrics=False) @@ -389,10 +428,16 @@ def pea_classification(params): print("Percentage of correct mappings",evaluation["Percentage correct"]) #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion return evaluation["Percentage correct"] - +def simple_test(params): + return params[0] data_moons = data1.copy() data_moons.set_name('Moon_Set') +data = data_moons.copy() classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] -HPO = HP_Optimization(pea_classification, classification_space) +simple_space = [["interval", 6.456, 10.2123]] +HPO = HP_Optimization(simple_test, simple_space) #HPO.perform_evaluation_at([0.00242204, 0, 1, 0]) -HPO.perform_BO_classification(20) +#HPO.perform_BO(20) +#print(HPO.cart_prod_hp_space(5)) +HPO.perform_BO(5) + From 18960ae022a2724dfb46cbfe967ddcfa9a704d1b Mon Sep 17 00:00:00 2001 From: Sonja Date: Thu, 12 Aug 2021 10:34:20 +0200 Subject: [PATCH 34/70] Added optimization of "perform_classification_dimension_wise" and a function to check given hp_space --- sparseSpACE/HP_Optimization.py | 180 +++++++++++++++++++++------------ 1 file changed, 118 insertions(+), 62 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index c0f8c6d..8eb3522 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -14,6 +14,30 @@ class HP_Optimization: def __init__(self, function, hp_space): self.function = function self.hp_space = hp_space + self.check_hp_space() + + def check_hp_space(self): + if(len(self.hp_space)<1): + print("Please enter values for hp_space! Using default value ['list', 1], but will likely throw errors") + hp_space.append(["list", 1]) + return 0 + for i in range(0, len(self.hp_space)): + if(len(self.hp_space[i])<2): + print("Please enter at least a descriptor and one value for hp_space index " + str(i)) + print("Default value ['list', 1] will be used") + hp_space[i] = ["list", 1] + elif(self.hp_space[i][0]=="interval"): + if(len(self.hp_space[i])<3): + self.hp_space[i].append(self.hp_space[i][1]) + elif(self.hp_space[i][1]>self.hp_space[i][2]): + self.hp_space[i] = ["interval", self.hp_space[i][2], self.hp_space[i][1]] + else: + self.hp_space[i] = ["interval", self.hp_space[i][1], self.hp_space[i][2]] + elif(self.hp_space[i][0]=="list"): + self.hp_space[i] = self.hp_space[i] + else: + print("Unknown descriptor for hp_space at index " + str(i) +". Using ['list', ]") + self.hp_space[i] = ["list", self.hp_space[i][1]] #performs Grid Optimization def perform_GO(self, int_levels): @@ -49,27 +73,33 @@ def perform_GO(self, int_levels): #+ " and time = " + str(best_time)) def cart_prod_hp_space(self, interval_levels): - res = [] - for i in range (0, len(self.hp_space)): - new = [] - if(len(self.hp_space[i])<3): - print("please enter enough values for hp_space") - new.append(1) - elif(self.hp_space[i][0] == "interval"): - h = self.hp_space[i][2]-self.hp_space[i][1] - dh = h /interval_levels - #currently adds 1 more than interval_levels so that both borders are included - is there a better way to do this? - for j in range (0, interval_levels+1): - new.append(self.hp_space[i][1]+j*dh) - elif(self.hp_space[i][0] == "list"): - new = self.hp_space[i].copy() - new.remove("list") - else: - print("please enter valid types for hp_space") - new.append[1] - res.append(new) - res_cart = list(itertools.product(*res)) - return res_cart + res = [] + for i in range (0, len(self.hp_space)): + new = [] + if(len(self.hp_space[i])<2): + print("please at least 1 value for hp_space list") + new.append(1) + elif(self.hp_space[i][0] == "list"): + new = self.hp_space[i].copy() + new.remove("list") + elif(len(self.hp_space[i])<3): + print("please enter at least 2 values for hp_space interval. Using the value given") + new.append(self.hp_space[i][1]) + elif(self.hp_space[i][0] == "interval"): + h = self.hp_space[i][2]-self.hp_space[i][1] + dh = h /interval_levels + #currently adds 1 more than interval_levels so that both borders are included - is there a better way to do this? + if(h == 0): + new.append(self.hp_space[i][1]) + else: + for j in range (0, interval_levels+1): + new.append(self.hp_space[i][1]+j*dh) + else: + print("please enter valid types for hp_space! Using the first value given") + new.append(self.hp_space[i][1]) + res.append(new) + res_cart = list(itertools.product(*res)) + return res_cart #performs random optimization def perform_RO(self, amt_it: int): @@ -202,18 +232,18 @@ def get_cov_matrix(self, C_x, cur_length: int): def get_cov_vector(self, ev_x, new_x, cur_length: int): #somehow the rest of the function would still work even if x is wrong size, but no idea why / what it does. Better to check if(len(ev_x[0]) != len(new_x)): - print("Input x has the wrong size") + print("Input x for cov_vector has the wrong size") return None k_vec = np.zeros(len(ev_x)) for i in range (0, cur_length, 1): k_vec[i] = self.cov(ev_x[i], new_x) return k_vec - #maybe give method the non-rand values instead of hardcoding them? - def create_evidence_set(self, amt_it: int, dim_HP: int, is_random: bool = True, non_rand_values = None): + #creates evidence set using random values + def create_evidence_set(self, amt_it: int, dim_HP: int, non_rand_values = None): x = np.zeros(shape=(amt_it+dim_HP+1, dim_HP)) y = np.zeros((amt_it+dim_HP+1)) - if(is_random): + if(non_rand_values == None): for i in range(0, dim_HP+1, 1): new_x = self.create_random_x() while(self.check_if_in_array(new_x, x)): @@ -224,15 +254,9 @@ def create_evidence_set(self, amt_it: int, dim_HP: int, is_random: bool = True, #needs to be casted to int bc otherwise it's automatically float which doesn't work y[i] = self.perform_evaluation_at(x[i]) else: - #hard-code non-random values for testing - current threw error at it. 24 - #use seed for random values instead of hard coded values? -> get used seed - x[0] = [0.20213575, 0., 1., 0.] #[0.79125794, 0, 1, 0] - x[1] = [0.80125658, 0., 1., 0.] #[0.69819941, 1, 2, 0] - x[2] = [0.09898312, 1., 3., 1.] #[0.35823418, 0, 1, 0] - x[3] = [0.88249225, 1., 1., 0.] #[0.51043662, 1, 1, 0] - x[4] = [0.1321559, 1., 2., 1.] #[0.54776247, 0, 1, 0] - for i in range(0, dim_HP+1, 1): - y[i] = self.perform_evaluation_at(x[i]) + for j in range(0, dim_HP+1, 1): + x[j] = non_rand_values[j] + y[j] = self.perform_evaluation_at(x[j]) return x, y #returns a random x for the given purpose - needs search space @@ -241,17 +265,21 @@ def create_random_x(self): res = [] for i in range (0, len(self.hp_space)): new_x = 1 - if (len(self.hp_space[i])<3): - print("Too little Arguments! HP Space index " + str(i) + ": " + str(self.hp_space[i])) - elif (self.hp_space[i][0] == "interval"): - new_x = random.uniform(self.hp_space[i][1], self.hp_space[i][2]) + if (len(self.hp_space[i])<2): + print("Please enter at least 1 value for hp_space list!") elif (self.hp_space[i][0] == "list"): sel = self.hp_space[i].copy() sel.remove("list") new_x = random.choice(sel) print("hp_space[" + str(i) + "] is now " + str(self.hp_space[i])) + elif (len(self.hp_space[i])<3): + print("Please enter at least 2 values for hp_space interval! Using the 1 value given") + new_x = self.hp_space[i][1] + elif (self.hp_space[i][0] == "interval"): + new_x = random.uniform(self.hp_space[i][1], self.hp_space[i][2]) else: - print("Unknown type of space! Using default value 1 for index " + str(i)) + print("Unknown type of space! Using first value given for index " + str(i)) + new_x = self.hp_space[i][1] res.append(new_x) return res @@ -311,14 +339,10 @@ def get_new_beta_and_l(self, cur_beta: float, cur_amt_x, cur_x, C_x, C_y): def get_bounds(self): res = [] for i in range(0, len(self.hp_space)): - new = [0, 1] + new = [1, 1] #should one be able to enter just one value for "list"?? - if(len(self.hp_space[i])<3): - print("please enter a valid hp_space") - elif(self.hp_space[i][0] == "interval"): - new = self.hp_space[i].copy() - new.remove("interval") - new = [new[0], new[1]] + if(len(self.hp_space[i])<2): + print("please enter at least 1 value for hp_space list. Using default values [1, 1]") elif(self.hp_space[i][0] == "list"): max = self.hp_space[i][1] min = max @@ -328,8 +352,16 @@ def get_bounds(self): if(self.hp_space[i][j]>max): max = self.hp_space[i][j] new = [min, max] + elif(len(self.hp_space[i])<3): + print("please enter at least 2 values for hp_space interval. Using the one value given") + new = [hp_space[i][1], hp_space[i][1]] + elif(self.hp_space[i][0] == "interval"): + new = self.hp_space[i].copy() + new.remove("interval") + new = [new[0], new[1]] else: - print("please enter a valid hp_space") + print("please enter a valid hp_space. Using first value given") + new = [hp_space[i][1], hp_space[i][1]] res.append(new) return res @@ -370,16 +402,8 @@ def round_x(self, x): new_x_rd = [] for i in range (0, len(self.hp_space)): new_x = 1 - if(len(self.hp_space[i])<3): - print("Too little Arguments! HP Space index " + str(i) + ": " + str(self.hp_space[i])) - print("Too little arguments in HP Space! Using default value 1 for index " + str(i)) - elif(self.hp_space[i][0] == "interval"): - if(x[i]self.hp_space[i][2]): - new_x = self.hp_space[i][2] - else: - new_x = x[i] + if(len(self.hp_space[i])<2): + print("Please enter at least 1 value for hp_space list. Index: " + str(i) + ": " + str(self.hp_space[i])) elif(self.hp_space[i][0] == "list"): new_x = self.hp_space[i][1] dis = abs(x[i]-self.hp_space[i][1]) @@ -389,8 +413,19 @@ def round_x(self, x): if(cur_dis < dis): dis = cur_dis new_x = self.hp_space[i][j] + elif(len(self.hp_space[i])<3): + print("Please enter at least 2 values for hp_space interval. Using the one value given") + new_x = self.hp_space[i][1] + elif(self.hp_space[i][0] == "interval"): + if(x[i]self.hp_space[i][2]): + new_x = self.hp_space[i][2] + else: + new_x = x[i] else: - print("Unknown type of space! Using default value 1 for index " + str(i)) + print("Unknown type of space! Using first value given for index " + str(i)) + new_x = self.hp_space[i][1] new_x_rd.append(new_x) return new_x_rd @@ -428,16 +463,37 @@ def pea_classification(params): print("Percentage of correct mappings",evaluation["Percentage correct"]) #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion return evaluation["Percentage correct"] +def pea_classification_dimension_wise(params): + if(len(params)<4): + print("too little params for pea_classification. Returning 0.0") + return 0.0 + params = [float(params[0]), int(params[1]), int(params[2]), int(params[3])] + #dataset_blobs = deml.datasets.make_blobs(n_samples=samples, n_features=dimension, centers=labels, random_state=1) + #data_blobs = deml.DataSet(dataset_blobs, name='Blobs_Set') + #cur_data = data_blobs.copy() + #dataset_moons = deml.datasets.make_moons(n_samples=samples, noise=0.15, random_state=1) + #data_moons = deml.DataSet(sklearn_dataset, name='data_moons') + cur_data = data.copy() + #should implement a smoother way to put in the data set + classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) + classification.perform_classification_dimension_wise(masslumping=params[1], lambd=params[0], minimum_level=params[2], maximum_level=5, one_vs_others=params[3], print_metrics=False) + evaluation = classification.evaluate() + print("Percentage of correct mappings",evaluation["Percentage correct"]) + #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion + return evaluation["Percentage correct"] + def simple_test(params): return params[0] data_moons = data1.copy() data_moons.set_name('Moon_Set') data = data_moons.copy() classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] -simple_space = [["interval", 6.456, 10.2123]] -HPO = HP_Optimization(simple_test, simple_space) +simple_space = [["interval", 1, 0], ["liste", 1, 3, 4], ["interval", 2], ["list", 0, 1]] +HPO = HP_Optimization(pea_classification_dimension_wise, simple_space) #HPO.perform_evaluation_at([0.00242204, 0, 1, 0]) #HPO.perform_BO(20) #print(HPO.cart_prod_hp_space(5)) -HPO.perform_BO(5) - +HPO.perform_GO(5) +#print(HPO.round_x([-1, -1, -1, -1])) +#HPO.check_hp_space() +#print(HPO.hp_space) From 06976bf45c8ba90aa6aac7c56ee40462dd500507 Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 17 Aug 2021 17:06:00 +0200 Subject: [PATCH 35/70] for easier cp-pst --- sparseSpACE/HP_Optimization.py | 93 ++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 39 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 8eb3522..b1d2176 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -1,4 +1,5 @@ import sparseSpACE.DEMachineLearning as deml +import sparseSpACE import numpy as np import scipy as sp import math @@ -130,14 +131,14 @@ def perform_evaluation_at(self, params): #performs Bayesian Optimization #Note: HP Input of classification is (lambd:float, massl:bool, min_lv:int <4, one_vs_others:bool) - def perform_BO(self, amt_it: int, ev_is_rand: bool = True): + def perform_BO(self, amt_it: int): #notes how many x values currently are in the evidence set - starts with amt_HP+1 amt_HP = len(self.hp_space) cur_amt_x: int = amt_HP+1 x_ret = None y_ret = None - print("I should do Bayesian Optimization for " + str(amt_it) + " iterations, ev_is_rand is " + str(ev_is_rand) + " !") - C = self.create_evidence_set(amt_it, amt_HP, ev_is_rand) + print("I should do Bayesian Optimization for " + str(amt_it) + " iterations!") + C = self.create_evidence_set(amt_it, amt_HP) C_x=C[0] print("C_x: " + str(C_x)) C_y=C[1] @@ -240,23 +241,19 @@ def get_cov_vector(self, ev_x, new_x, cur_length: int): return k_vec #creates evidence set using random values - def create_evidence_set(self, amt_it: int, dim_HP: int, non_rand_values = None): + def create_evidence_set(self, amt_it: int, dim_HP: int): x = np.zeros(shape=(amt_it+dim_HP+1, dim_HP)) y = np.zeros((amt_it+dim_HP+1)) - if(non_rand_values == None): - for i in range(0, dim_HP+1, 1): + for i in range(0, dim_HP+1, 1): + new_x = self.create_random_x() + while(self.check_if_in_array(new_x, x)): new_x = self.create_random_x() - while(self.check_if_in_array(new_x, x)): - new_x = self.create_random_x() - x[i] = new_x - #evaluating takes quite some time, especially for min_lv>1 - #y[i] = perform_evaluation_at(data, new_x[0], new_x[1], new_x[2], new_x[3]) - #needs to be casted to int bc otherwise it's automatically float which doesn't work - y[i] = self.perform_evaluation_at(x[i]) - else: - for j in range(0, dim_HP+1, 1): - x[j] = non_rand_values[j] - y[j] = self.perform_evaluation_at(x[j]) + x[i] = new_x + #evaluating takes quite some time, especially for min_lv>1 + #y[i] = perform_evaluation_at(data, new_x[0], new_x[1], new_x[2], new_x[3]) + #needs to be casted to int bc otherwise it's automatically float which doesn't work + y[i] = self.perform_evaluation_at(x[i]) + return x, y #returns a random x for the given purpose - needs search space @@ -429,17 +426,30 @@ def round_x(self, x): new_x_rd.append(new_x) return new_x_rd +class Optimize_Classification: + def __init__{self, data, samples: int = 500, dimension: int = 2, labels: int = 6, max_lv: int = 3, max_evals: int = 256} + self.data = data + self.samples = samples + self.dimension = dimension + self.labels = labels + self.max_lv = max_lv + self.max_evals = max_evals + #Basically copied code from Tutorial_DEMachineLearning #prepare Dataset samples = 500 -dimension = 2 +dimension = 5 labels = 6 #vllt anderes dataset ausprobieren?, vllt höhere dimensionen, bis zu dim=10. Note: moons immer 2d, aber z.B. #gaussian quantiles, classification.. kann man einstellen -sklearn_dataset = deml.datasets.make_moons(n_samples=samples, noise=0.15, random_state=1) -data1 = deml.DataSet(sklearn_dataset, name='Input_Set') +dataset_moons = deml.datasets.make_moons(n_samples=samples, noise=0.15, random_state=1) +data_moons = deml.DataSet(dataset_moons, name='Moon_Set') dataset_blobs = deml.datasets.make_blobs(n_samples=samples, n_features=dimension, centers=labels) data_blobs = deml.DataSet(dataset_blobs, name='Blobs_Set') +dataset_circles = deml.datasets.make_circles(n_samples=samples, noise=0.05) +data_circles = deml.DataSet(dataset_circles, name='Circle_Set') +dataset_class = deml.datasets.make_classification(n_samples=samples, n_features=dimension, n_redundant=0, n_clusters_per_class=1, n_informative=2, n_classes=(labels if labels < 4 else 4)) +data_class = deml.DataSet(dataset_class, name='Classification_Set') #perform_evaluation_at(data_moons, 0.00242204, 0, 1, 0) #.98 evaluation! #perform_BO_classification(data_blobs, 20) #perform_RO_classification(data_moons, 10, dimension) @@ -464,36 +474,41 @@ def pea_classification(params): #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion return evaluation["Percentage correct"] def pea_classification_dimension_wise(params): - if(len(params)<4): + if(len(params)<3): print("too little params for pea_classification. Returning 0.0") + #use default values? return 0.0 - params = [float(params[0]), int(params[1]), int(params[2]), int(params[3])] - #dataset_blobs = deml.datasets.make_blobs(n_samples=samples, n_features=dimension, centers=labels, random_state=1) - #data_blobs = deml.DataSet(dataset_blobs, name='Blobs_Set') - #cur_data = data_blobs.copy() - #dataset_moons = deml.datasets.make_moons(n_samples=samples, noise=0.15, random_state=1) - #data_moons = deml.DataSet(sklearn_dataset, name='data_moons') + #lambd, massl, min_lv, one_vs_others / error calc, margin(float 0-1), rebalancing (bool), use_relative_surplus (bool) + params = [float(params[0]), int(params[1]), int(params[2]), int(params[3]), int(params[4]), float(params[5]), int(params[6])] cur_data = data.copy() - #should implement a smoother way to put in the data set + error_calculator=sparseSpACE.ErrorCalculator.ErrorCalculatorSingleDimMisclassificationGlobal() + ec = None + ovo = False + if(params[3] == 2): + ec = error_calculator + ovo = True + elif(params[3] == 1): + ovo = True classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) - classification.perform_classification_dimension_wise(masslumping=params[1], lambd=params[0], minimum_level=params[2], maximum_level=5, one_vs_others=params[3], print_metrics=False) + classification.perform_classification_dimension_wise(masslumping=params[1], lambd=params[0], minimum_level=params[2], maximum_level=3, one_vs_others=ovo, error_calculator = ec, margin = params[4], rebalancing = params[5], use_relative_surplus = params[6], print_metrics=False) evaluation = classification.evaluate() print("Percentage of correct mappings",evaluation["Percentage correct"]) #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion return evaluation["Percentage correct"] +def optimize_classification_dim_wise(data_in, max_lv: int = 3, max_evals: int = 256): + global data + data = data_in + class_dim_wise_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2], ["list", 0, 1, 2], ["interval", 0, 1], ["list", 0, 1], ["list", 0, 1]] + HPO = HP_Optimization(pea_classification_dimension_wise, class_dim_wise_space) + HPO.perform_BO(5) def simple_test(params): return params[0] -data_moons = data1.copy() -data_moons.set_name('Moon_Set') data = data_moons.copy() + classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] +class_dim_wise_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] simple_space = [["interval", 1, 0], ["liste", 1, 3, 4], ["interval", 2], ["list", 0, 1]] -HPO = HP_Optimization(pea_classification_dimension_wise, simple_space) -#HPO.perform_evaluation_at([0.00242204, 0, 1, 0]) -#HPO.perform_BO(20) -#print(HPO.cart_prod_hp_space(5)) -HPO.perform_GO(5) -#print(HPO.round_x([-1, -1, -1, -1])) -#HPO.check_hp_space() -#print(HPO.hp_space) +#HPO = HP_Optimization(pea_classification_dimension_wise, class_dim_wise_space) +#HPO.perform_BO(5) +optimize_classification_dim_wise(data_blobs) From 6c7a88d53d755dbd121fb9d7ca8347130d2643d8 Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 17 Aug 2021 17:44:03 +0200 Subject: [PATCH 36/70] added new class for functions to be optimized --- sparseSpACE/HP_Optimization.py | 59 ++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index b1d2176..0f8ee88 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -20,13 +20,13 @@ def __init__(self, function, hp_space): def check_hp_space(self): if(len(self.hp_space)<1): print("Please enter values for hp_space! Using default value ['list', 1], but will likely throw errors") - hp_space.append(["list", 1]) + self.hp_space.append(["list", 1]) return 0 for i in range(0, len(self.hp_space)): if(len(self.hp_space[i])<2): print("Please enter at least a descriptor and one value for hp_space index " + str(i)) print("Default value ['list', 1] will be used") - hp_space[i] = ["list", 1] + self.hp_space[i] = ["list", 1] elif(self.hp_space[i][0]=="interval"): if(len(self.hp_space[i])<3): self.hp_space[i].append(self.hp_space[i][1]) @@ -427,18 +427,56 @@ def round_x(self, x): return new_x_rd class Optimize_Classification: - def __init__{self, data, samples: int = 500, dimension: int = 2, labels: int = 6, max_lv: int = 3, max_evals: int = 256} + def __init__(self, data, samples: int = 500, dimension: int = 2, labels: int = 6, max_lv: int = 3, max_evals: int = 256): self.data = data self.samples = samples self.dimension = dimension self.labels = labels self.max_lv = max_lv self.max_evals = max_evals - + self.classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2], ["list", 0, 1]] + self.class_dim_wise_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2], ["list", 0, 1, 2], ["interval", 0, 1], ["list", 0, 1], ["list", 0, 1]] + + def pea_classification(self, params): + if(len(params)<4): + print("too little params for pea_classification. Returning 0.0") + return 0.0 + params = [params[0], int(params[1]), int(params[2]), int(params[3])] + cur_data = self.data.copy() + #should implement a smoother way to put in the data set + classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) + classification.perform_classification(masslumping=params[1], lambd=params[0], minimum_level=params[2], maximum_level=self.max_lv, one_vs_others=params[3], print_metrics=False) + evaluation = classification.evaluate() + print("Percentage of correct mappings",evaluation["Percentage correct"]) + #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion + return evaluation["Percentage correct"] + + def pea_classification_dimension_wise(self, params): + if(len(params)<4): + print("too little params for pea_classification. Returning 0.0") + return 0.0 + #lambd, massl, min_lv, one_vs_others / error calc, margin(float 0-1), rebalancing (bool), use_relative_surplus (bool) + params = [float(params[0]), int(params[1]), int(params[2]), int(params[3]), float(params[4]), int(params[5]), int(params[6])] + cur_data = self.data.copy() + error_calculator=sparseSpACE.ErrorCalculator.ErrorCalculatorSingleDimMisclassificationGlobal() + ec = None + ovo = False + if(params[3] == 2): + ec = error_calculator + ovo = True + elif(params[3] == 1): + ovo = True + classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) + classification.perform_classification_dimension_wise(masslumping=params[1], lambd=params[0], minimum_level=params[2], maximum_level=self.max_lv, one_vs_others=ovo, error_calculator = ec, margin = params[4], rebalancing = params[5], use_relative_surplus = params[6], print_metrics=False) + evaluation = classification.evaluate() + print("Percentage of correct mappings",evaluation["Percentage correct"]) + #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion + return evaluation["Percentage correct"] + #Basically copied code from Tutorial_DEMachineLearning #prepare Dataset samples = 500 -dimension = 5 +dimension = 2 labels = 6 #vllt anderes dataset ausprobieren?, vllt höhere dimensionen, bis zu dim=10. Note: moons immer 2d, aber z.B. #gaussian quantiles, classification.. kann man einstellen @@ -468,7 +506,7 @@ def pea_classification(params): cur_data = data.copy() #should implement a smoother way to put in the data set classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) - classification.perform_classification(masslumping=params[1], lambd=params[0], minimum_level=params[2], maximum_level=5, one_vs_others=params[3], print_metrics=False) + classification.perform_classification_dimension_wise(masslumping=params[1], lambd=params[0], minimum_level=params[2], maximum_level=3, one_vs_others=ovo, error_calculator = ec, margin = params[4], rebalancing = params[5], use_relative_surplus = params[6], print_metrics=False) evaluation = classification.evaluate() print("Percentage of correct mappings",evaluation["Percentage correct"]) #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion @@ -505,10 +543,11 @@ def optimize_classification_dim_wise(data_in, max_lv: int = 3, max_evals: int = def simple_test(params): return params[0] data = data_moons.copy() - +OC = Optimize_Classification(data = data_moons, max_lv = 2) classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] class_dim_wise_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] simple_space = [["interval", 1, 0], ["liste", 1, 3, 4], ["interval", 2], ["list", 0, 1]] -#HPO = HP_Optimization(pea_classification_dimension_wise, class_dim_wise_space) -#HPO.perform_BO(5) -optimize_classification_dim_wise(data_blobs) + +HPO = HP_Optimization(OC.pea_classification_dimension_wise, OC.class_dim_wise_space) +HPO.perform_BO(5) +#optimize_classification_dim_wise(data_blobs) From 3c35d0b8e7ea7786a9e08da34c6eb087c8064b71 Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 17 Aug 2021 18:25:43 +0200 Subject: [PATCH 37/70] Created a function to set data --- sparseSpACE/HP_Optimization.py | 95 +++++++--------------------------- 1 file changed, 19 insertions(+), 76 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 0f8ee88..6ab1627 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -427,16 +427,29 @@ def round_x(self, x): return new_x_rd class Optimize_Classification: - def __init__(self, data, samples: int = 500, dimension: int = 2, labels: int = 6, max_lv: int = 3, max_evals: int = 256): - self.data = data + def __init__(self, data_name: str = "moons", samples: int = 500, dimension: int = 2, labels: int = 6, max_lv: int = 3, max_evals: int = 256): self.samples = samples self.dimension = dimension self.labels = labels + self.data = self.create_data(data_name) self.max_lv = max_lv self.max_evals = max_evals self.classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2], ["list", 0, 1]] self.class_dim_wise_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2], ["list", 0, 1, 2], ["interval", 0, 1], ["list", 0, 1], ["list", 0, 1]] + def create_data(self, name: str = "moons"): + dataset = deml.datasets.make_moons(n_samples=self.samples, noise=0.15, random_state=1) + if(name == "blobs"): + dataset = deml.datasets.make_blobs(n_samples=self.samples, n_features=self.dimension, centers=self.labels) + elif(name == "circles"): + dataset = deml.datasets.make_circles(n_samples=self.samples, noise=0.05) + elif(name == "classification"): + dataset = deml.datasets.make_classification(n_samples=self.samples, n_features=self.dimension, n_redundant=0, n_clusters_per_class=1, n_informative=2, n_classes=(labels if labels < 4 else 4)) + else: + print("This is not a valid name for a dataset, using moons") + data = deml.DataSet(dataset, name="data_"+name) + return data + def pea_classification(self, params): if(len(params)<4): print("too little params for pea_classification. Returning 0.0") @@ -473,81 +486,11 @@ def pea_classification_dimension_wise(self, params): #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion return evaluation["Percentage correct"] -#Basically copied code from Tutorial_DEMachineLearning -#prepare Dataset -samples = 500 -dimension = 2 -labels = 6 -#vllt anderes dataset ausprobieren?, vllt höhere dimensionen, bis zu dim=10. Note: moons immer 2d, aber z.B. -#gaussian quantiles, classification.. kann man einstellen -dataset_moons = deml.datasets.make_moons(n_samples=samples, noise=0.15, random_state=1) -data_moons = deml.DataSet(dataset_moons, name='Moon_Set') -dataset_blobs = deml.datasets.make_blobs(n_samples=samples, n_features=dimension, centers=labels) -data_blobs = deml.DataSet(dataset_blobs, name='Blobs_Set') -dataset_circles = deml.datasets.make_circles(n_samples=samples, noise=0.05) -data_circles = deml.DataSet(dataset_circles, name='Circle_Set') -dataset_class = deml.datasets.make_classification(n_samples=samples, n_features=dimension, n_redundant=0, n_clusters_per_class=1, n_informative=2, n_classes=(labels if labels < 4 else 4)) -data_class = deml.DataSet(dataset_class, name='Classification_Set') -#perform_evaluation_at(data_moons, 0.00242204, 0, 1, 0) #.98 evaluation! -#perform_BO_classification(data_blobs, 20) -#perform_RO_classification(data_moons, 10, dimension) -#perform_evaluation_at(self, data_moons, 0.828603059876013, 0, 2, 1) -#parameters are in form lambd, masslump, minlv, one_vs_others -def pea_classification(params): - if(len(params)<4): - print("too little params for pea_classification. Returning 0.0") - return 0.0 - params = [params[0], int(params[1]), int(params[2]), int(params[3])] - #dataset_blobs = deml.datasets.make_blobs(n_samples=samples, n_features=dimension, centers=labels, random_state=1) - #data_blobs = deml.DataSet(dataset_blobs, name='Blobs_Set') - #cur_data = data_blobs.copy() - #dataset_moons = deml.datasets.make_moons(n_samples=samples, noise=0.15, random_state=1) - #data_moons = deml.DataSet(sklearn_dataset, name='data_moons') - cur_data = data.copy() - #should implement a smoother way to put in the data set - classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) - classification.perform_classification_dimension_wise(masslumping=params[1], lambd=params[0], minimum_level=params[2], maximum_level=3, one_vs_others=ovo, error_calculator = ec, margin = params[4], rebalancing = params[5], use_relative_surplus = params[6], print_metrics=False) - evaluation = classification.evaluate() - print("Percentage of correct mappings",evaluation["Percentage correct"]) - #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion - return evaluation["Percentage correct"] -def pea_classification_dimension_wise(params): - if(len(params)<3): - print("too little params for pea_classification. Returning 0.0") - #use default values? - return 0.0 - #lambd, massl, min_lv, one_vs_others / error calc, margin(float 0-1), rebalancing (bool), use_relative_surplus (bool) - params = [float(params[0]), int(params[1]), int(params[2]), int(params[3]), int(params[4]), float(params[5]), int(params[6])] - cur_data = data.copy() - error_calculator=sparseSpACE.ErrorCalculator.ErrorCalculatorSingleDimMisclassificationGlobal() - ec = None - ovo = False - if(params[3] == 2): - ec = error_calculator - ovo = True - elif(params[3] == 1): - ovo = True - classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) - classification.perform_classification_dimension_wise(masslumping=params[1], lambd=params[0], minimum_level=params[2], maximum_level=3, one_vs_others=ovo, error_calculator = ec, margin = params[4], rebalancing = params[5], use_relative_surplus = params[6], print_metrics=False) - evaluation = classification.evaluate() - print("Percentage of correct mappings",evaluation["Percentage correct"]) - #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion - return evaluation["Percentage correct"] -def optimize_classification_dim_wise(data_in, max_lv: int = 3, max_evals: int = 256): - global data - data = data_in - class_dim_wise_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2], ["list", 0, 1, 2], ["interval", 0, 1], ["list", 0, 1], ["list", 0, 1]] - HPO = HP_Optimization(pea_classification_dimension_wise, class_dim_wise_space) - HPO.perform_BO(5) - +#anderes dataset ausprobieren?, vllt höhere dimensionen, bis zu dim=10. Note: moons immer 2d, aber z.B. def simple_test(params): return params[0] -data = data_moons.copy() -OC = Optimize_Classification(data = data_moons, max_lv = 2) -classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] -class_dim_wise_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] simple_space = [["interval", 1, 0], ["liste", 1, 3, 4], ["interval", 2], ["list", 0, 1]] -HPO = HP_Optimization(OC.pea_classification_dimension_wise, OC.class_dim_wise_space) -HPO.perform_BO(5) -#optimize_classification_dim_wise(data_blobs) +OC = Optimize_Classification(data_name = "circles", dimension = 2) +HPO = HP_Optimization(OC.pea_classification, OC.classification_space) +HPO.perform_GO(100) From 8f22eaaf11fae21974ec62e9ed416212a0004152 Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 17 Aug 2021 18:49:47 +0200 Subject: [PATCH 38/70] added more data_sets to create_data method --- sparseSpACE/HP_Optimization.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 6ab1627..c519846 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -445,6 +445,16 @@ def create_data(self, name: str = "moons"): dataset = deml.datasets.make_circles(n_samples=self.samples, noise=0.05) elif(name == "classification"): dataset = deml.datasets.make_classification(n_samples=self.samples, n_features=self.dimension, n_redundant=0, n_clusters_per_class=1, n_informative=2, n_classes=(labels if labels < 4 else 4)) + elif(name == "gaussian_quantiles"): + dataset = deml.datasets.make_gaussian_quantiles(n_samples=self.samples, n_features=self.dimension, n_classes=self.labels) + elif(name == "digits"): + dataset = deml.datasets.load_digits(return_X_y=True) # hint: try only with max_level <= 3 + elif(name == "iris"): + dataset = deml.datasets.load_iris(return_X_y=True) + elif(name == "breast_cancer"): + dataset = deml.datasets.load_breast_cancer(return_X_y=True) # hint: try only with max_level <= 4 + elif(name == "wine"): + dataset = deml.datasets.load_wine(return_X_y=True) else: print("This is not a valid name for a dataset, using moons") data = deml.DataSet(dataset, name="data_"+name) From ef32c5197730ee807db1a274f0cc1341bae1e074 Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 17 Aug 2021 20:01:51 +0200 Subject: [PATCH 39/70] Testing different data sets --- sparseSpACE/HP_Optimization.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index c519846..84abb9c 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -434,8 +434,8 @@ def __init__(self, data_name: str = "moons", samples: int = 500, dimension: int self.data = self.create_data(data_name) self.max_lv = max_lv self.max_evals = max_evals - self.classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2], ["list", 0, 1]] - self.class_dim_wise_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2], ["list", 0, 1, 2], ["interval", 0, 1], ["list", 0, 1], ["list", 0, 1]] + self.classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1], ["list", 0, 1]] + self.class_dim_wise_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1], ["list", 0, 1, 2], ["interval", 0, 1], ["list", 0, 1], ["list", 0, 1]] def create_data(self, name: str = "moons"): dataset = deml.datasets.make_moons(n_samples=self.samples, noise=0.15, random_state=1) @@ -501,6 +501,6 @@ def simple_test(params): return params[0] simple_space = [["interval", 1, 0], ["liste", 1, 3, 4], ["interval", 2], ["list", 0, 1]] -OC = Optimize_Classification(data_name = "circles", dimension = 2) +OC = Optimize_Classification(data_name = "iris", dimension = 2) HPO = HP_Optimization(OC.pea_classification, OC.classification_space) -HPO.perform_GO(100) +HPO.perform_BO(3) From 321acb4d9479340f07afccaad1f3e757f815fabe Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 17 Aug 2021 20:09:03 +0200 Subject: [PATCH 40/70] fixed a mistake --- sparseSpACE/HP_Optimization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 84abb9c..e4ea19b 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -444,7 +444,7 @@ def create_data(self, name: str = "moons"): elif(name == "circles"): dataset = deml.datasets.make_circles(n_samples=self.samples, noise=0.05) elif(name == "classification"): - dataset = deml.datasets.make_classification(n_samples=self.samples, n_features=self.dimension, n_redundant=0, n_clusters_per_class=1, n_informative=2, n_classes=(labels if labels < 4 else 4)) + dataset = deml.datasets.make_classification(n_samples=self.samples, n_features=self.dimension, n_redundant=0, n_clusters_per_class=1, n_informative=2, n_classes=(self.labels if self.labels < 4 else 4)) elif(name == "gaussian_quantiles"): dataset = deml.datasets.make_gaussian_quantiles(n_samples=self.samples, n_features=self.dimension, n_classes=self.labels) elif(name == "digits"): From 28c1bb3e7ca5d90cf322c22313ce7d6334363013 Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 17 Aug 2021 20:18:02 +0200 Subject: [PATCH 41/70] fixed 2 more little mistakes --- sparseSpACE/HP_Optimization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index e4ea19b..4195c96 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -351,14 +351,14 @@ def get_bounds(self): new = [min, max] elif(len(self.hp_space[i])<3): print("please enter at least 2 values for hp_space interval. Using the one value given") - new = [hp_space[i][1], hp_space[i][1]] + new = [self.hp_space[i][1], self.hp_space[i][1]] elif(self.hp_space[i][0] == "interval"): new = self.hp_space[i].copy() new.remove("interval") new = [new[0], new[1]] else: print("please enter a valid hp_space. Using first value given") - new = [hp_space[i][1], hp_space[i][1]] + new = [self.hp_space[i][1], self.hp_space[i][1]] res.append(new) return res From 397173664dcea638e861bcfc3548f130f22911b4 Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 17 Aug 2021 22:24:25 +0200 Subject: [PATCH 42/70] added "interval_int" as option for hp_space --- sparseSpACE/HP_Optimization.py | 44 ++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 4195c96..54b5b28 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -34,6 +34,13 @@ def check_hp_space(self): self.hp_space[i] = ["interval", self.hp_space[i][2], self.hp_space[i][1]] else: self.hp_space[i] = ["interval", self.hp_space[i][1], self.hp_space[i][2]] + elif(self.hp_space[i][0]=="interval_int"): + if(len(self.hp_space[i])<3): + self.hp_space[i] = ["interval_int", int(self.hp_space[i][1]), int(self.hp_space[i][1])] + elif(self.hp_space[i][1]>self.hp_space[i][2]): + self.hp_space[i] = ["interval_int", int(self.hp_space[i][2]), int(self.hp_space[i][1])] + else: + self.hp_space[i] = ["interval_int", int(self.hp_space[i][1]), int(self.hp_space[i][2])] elif(self.hp_space[i][0]=="list"): self.hp_space[i] = self.hp_space[i] else: @@ -95,6 +102,17 @@ def cart_prod_hp_space(self, interval_levels): else: for j in range (0, interval_levels+1): new.append(self.hp_space[i][1]+j*dh) + elif(self.hp_space[i][0] == "interval_int"): + h = int(self.hp_space[i][2]-self.hp_space[i][1]) + dh = h /interval_levels + #currently adds 1 more than interval_levels so that both borders are included - is there a better way to do this? + if(h == 0): + new.append(int(self.hp_space[i][1])) + else: + if(dh<1): + dh = 1 + for j in range (0, self.hp_space[i][2]+1, int(dh)): + new.append(int(self.hp_space[i][1]+j*dh)) else: print("please enter valid types for hp_space! Using the first value given") new.append(self.hp_space[i][1]) @@ -274,6 +292,8 @@ def create_random_x(self): new_x = self.hp_space[i][1] elif (self.hp_space[i][0] == "interval"): new_x = random.uniform(self.hp_space[i][1], self.hp_space[i][2]) + elif (self.hp_space[i][0] == "interval_int"): + new_x = int(random.randrange(int(self.hp_space[i][1]), int(self.hp_space[i][2]))) else: print("Unknown type of space! Using first value given for index " + str(i)) new_x = self.hp_space[i][1] @@ -356,6 +376,10 @@ def get_bounds(self): new = self.hp_space[i].copy() new.remove("interval") new = [new[0], new[1]] + elif(self.hp_space[i][0] == "interval_int"): + new = self.hp_space[i].copy() + new.remove("interval_int") + new = [new[0], new[1]] else: print("please enter a valid hp_space. Using first value given") new = [self.hp_space[i][1], self.hp_space[i][1]] @@ -420,6 +444,13 @@ def round_x(self, x): new_x = self.hp_space[i][2] else: new_x = x[i] + elif(self.hp_space[i][0] == "interval_int"): + if(x[i]self.hp_space[i][2]): + new_x = int(self.hp_space[i][2]) + else: + new_x = round(x[i]) else: print("Unknown type of space! Using first value given for index " + str(i)) new_x = self.hp_space[i][1] @@ -478,7 +509,7 @@ def pea_classification_dimension_wise(self, params): if(len(params)<4): print("too little params for pea_classification. Returning 0.0") return 0.0 - #lambd, massl, min_lv, one_vs_others / error calc, margin(float 0-1), rebalancing (bool), use_relative_surplus (bool) + #lambd, massl, min_lv, one_vs_others / error calc, margin(float 0-1), rebalancing (bool), use_relative_surplus (bool), params = [float(params[0]), int(params[1]), int(params[2]), int(params[3]), float(params[4]), int(params[5]), int(params[6])] cur_data = self.data.copy() error_calculator=sparseSpACE.ErrorCalculator.ErrorCalculatorSingleDimMisclassificationGlobal() @@ -499,8 +530,11 @@ def pea_classification_dimension_wise(self, params): #anderes dataset ausprobieren?, vllt höhere dimensionen, bis zu dim=10. Note: moons immer 2d, aber z.B. def simple_test(params): return params[0] -simple_space = [["interval", 1, 0], ["liste", 1, 3, 4], ["interval", 2], ["list", 0, 1]] +simple_space = [["interval_int", 3.4, 0], ["list", 1, 3, 4], ["interval", 2], ["list", 0, 1]] -OC = Optimize_Classification(data_name = "iris", dimension = 2) -HPO = HP_Optimization(OC.pea_classification, OC.classification_space) -HPO.perform_BO(3) +#OC = Optimize_Classification(data_name = "iris", dimension = 2) +#HPO = HP_Optimization(OC.pea_classification, OC.classification_space) +HPO = HP_Optimization(simple_test, simple_space) +print(HPO.hp_space) +print(HPO.cart_prod_hp_space(2)) +#HPO.perform_BO(3) From 0a36b3c087b4575c5c6fdaf97e8fea5322170ea3 Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 17 Aug 2021 22:54:07 +0200 Subject: [PATCH 43/70] tested interval_int and fixed minor mistakes --- sparseSpACE/HP_Optimization.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 54b5b28..a82a02a 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -36,11 +36,11 @@ def check_hp_space(self): self.hp_space[i] = ["interval", self.hp_space[i][1], self.hp_space[i][2]] elif(self.hp_space[i][0]=="interval_int"): if(len(self.hp_space[i])<3): - self.hp_space[i] = ["interval_int", int(self.hp_space[i][1]), int(self.hp_space[i][1])] + self.hp_space[i] = ["interval_int", round(self.hp_space[i][1]), round(self.hp_space[i][1])] elif(self.hp_space[i][1]>self.hp_space[i][2]): - self.hp_space[i] = ["interval_int", int(self.hp_space[i][2]), int(self.hp_space[i][1])] + self.hp_space[i] = ["interval_int", round(self.hp_space[i][2]), round(self.hp_space[i][1])] else: - self.hp_space[i] = ["interval_int", int(self.hp_space[i][1]), int(self.hp_space[i][2])] + self.hp_space[i] = ["interval_int", round(self.hp_space[i][1]), round(self.hp_space[i][2])] elif(self.hp_space[i][0]=="list"): self.hp_space[i] = self.hp_space[i] else: @@ -293,7 +293,8 @@ def create_random_x(self): elif (self.hp_space[i][0] == "interval"): new_x = random.uniform(self.hp_space[i][1], self.hp_space[i][2]) elif (self.hp_space[i][0] == "interval_int"): - new_x = int(random.randrange(int(self.hp_space[i][1]), int(self.hp_space[i][2]))) + new_x = int(random.randrange(int(self.hp_space[i][1]), int(self.hp_space[i][2]+1))) + #max+1 bc second border value is not included else: print("Unknown type of space! Using first value given for index " + str(i)) new_x = self.hp_space[i][1] @@ -530,11 +531,12 @@ def pea_classification_dimension_wise(self, params): #anderes dataset ausprobieren?, vllt höhere dimensionen, bis zu dim=10. Note: moons immer 2d, aber z.B. def simple_test(params): return params[0] -simple_space = [["interval_int", 3.4, 0], ["list", 1, 3, 4], ["interval", 2], ["list", 0, 1]] +simple_space = [["interval_int", 3.4, 0.7], ["list", 1, 3, 4], ["interval", 2], ["list", 0, 1]] #OC = Optimize_Classification(data_name = "iris", dimension = 2) #HPO = HP_Optimization(OC.pea_classification, OC.classification_space) HPO = HP_Optimization(simple_test, simple_space) print(HPO.hp_space) -print(HPO.cart_prod_hp_space(2)) +print(HPO.round_x([2.9, 2, 1, 2])) +#print(HPO.cart_prod_hp_space(2)) #HPO.perform_BO(3) From 6a784482c344c33ded48a46ed6f55230fce343d4 Mon Sep 17 00:00:00 2001 From: Sonja Date: Wed, 18 Aug 2021 10:20:43 +0200 Subject: [PATCH 44/70] added "f_max" as value where optimization automatically stops --- sparseSpACE/HP_Optimization.py | 86 ++++++++++++++++++++++------------ 1 file changed, 57 insertions(+), 29 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index a82a02a..f089885 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -12,10 +12,11 @@ #the function on which HP should be Optimized and search space for the HPs #space is in form [["interval", , ], ["list", , , ,...], ....] class HP_Optimization: - def __init__(self, function, hp_space): + def __init__(self, function, hp_space, f_max = None): self.function = function self.hp_space = hp_space self.check_hp_space() + self.f_max = f_max def check_hp_space(self): if(len(self.hp_space)<1): @@ -48,7 +49,7 @@ def check_hp_space(self): self.hp_space[i] = ["list", self.hp_space[i][1]] #performs Grid Optimization - def perform_GO(self, int_levels): + def perform_GO(self, interval_levels = 4, search_space = None): #sklearn_dataset = deml.datasets.make_moons(n_samples=samples, noise=0.15) #data = deml.DataSet(sklearn_dataset, name='Input_Set') #for storing the current best evaluation and time @@ -63,25 +64,32 @@ def perform_GO(self, int_levels): #Note: lambd vllt besser logarithmisch, sehr kleine Werte oft besser (1/10, 1/100, 1/1000) #für for-schleife karthesisches produkt aus HO um alle tupel zu bilden und über diese tupel zu iterieren für beliebig dimensional #Note: lambd vor Allem sinnvoll wenn masslumping aus ist, sollte kaum unterschied machen wenn an - search_space = self.cart_prod_hp_space(int_levels) + if(search_space == None): + search_space = self.cart_prod_hp_space(interval_levels) for x in search_space: #use "perform evaluation at" #cur_time = classification._time_used #print ("current time needed = " + str(cur_time)) cur_evaluation = self.perform_evaluation_at(x) - print("Percentage of correct mappings",cur_evaluation) + print("Current Evaluation: ",cur_evaluation) #if best_evaluation == None or cur_evaluation > best_evaluation or (cur_evaluation == best_evaluation and (best_time == None or best_time>cur_time)): if(best_evaluation == None or cur_evaluation>best_evaluation): best_evaluation = cur_evaluation best_x = x - print("Best evaluation is now " + str(best_evaluation) + " at " + str(x)) - else: print("Best evaluation is still " + str(best_evaluation) + " at " + str(x)) + print("Best evaluation is now " + str(best_evaluation) + " at " + str(best_x)) + else: print("Best evaluation is still " + str(best_evaluation) + " at " + str(best_x)) print ("END OF CURRENT EVALUATION \n") - print("In the end, best evaluation is " + str(best_evaluation) + " at " + str(x)) + if(best_evaluation==self.f_max): + print("We've reached the specified maximum of f. Stopping GO.") + break + print("In the end, best evaluation is " + str(best_evaluation) + " at " + str(best_x)) + return best_x, best_evaluation #+ " and time = " + str(best_time)) def cart_prod_hp_space(self, interval_levels): res = [] + if (interval_levels == 0 or interval_levels == None): + interval_levels = 1 for i in range (0, len(self.hp_space)): new = [] if(len(self.hp_space[i])<2): @@ -111,8 +119,9 @@ def cart_prod_hp_space(self, interval_levels): else: if(dh<1): dh = 1 - for j in range (0, self.hp_space[i][2]+1, int(dh)): - new.append(int(self.hp_space[i][1]+j*dh)) + #may yield different values than expected due to rounding + for j in range (self.hp_space[i][1], self.hp_space[i][2]+1, int(dh)): + new.append(int(j)) else: print("please enter valid types for hp_space! Using the first value given") new.append(self.hp_space[i][1]) @@ -125,12 +134,17 @@ def perform_RO(self, amt_it: int): best_evaluation = 0 best_x = None for i in range (0, amt_it, 1): + print("Random step " + str(i)) x = self.create_random_x() new_eval = self.perform_evaluation_at(x) if new_eval>best_evaluation: best_x = x best_evaluation = new_eval + if(best_evaluation == self.f_max): + print("We've reached the specified maximum of f. Stopping RO at iteration " + str(i)) + break print("Best evaluation in " + str(amt_it) + " random steps: " + str(best_evaluation) + " at " + str(best_x)) + return best_x, best_evaluation #returns evaluation for certain Parameters on a certain data set def perform_evaluation_at(self, params): @@ -181,18 +195,22 @@ def perform_BO(self, amt_it: int): print("new x: " + str(new_x)) new_x_rd = self.round_x(new_x) old_x_rd = None + old_beta_and_l = [beta, l] print("new x rd: " + str(new_x_rd)) #erstelle neues beta und l und berechne neuen wert damit + amt_tries = 0 while(self.check_if_in_array(new_x_rd, C_x)): print("!!!!!!!!!!!Need new beta and l") old_x_rd = new_x_rd.copy() print("old x rd was: " + str(old_x_rd)) - beta_and_l = self.get_new_beta_and_l(beta, cur_amt_x, new_x, C_x, C_y) - new_x = self.acq_x(beta_and_l[0], beta_and_l[1], C_x, C_y, cur_amt_x) + new_beta_and_l = self.get_new_beta_and_l(beta, cur_amt_x, new_x, C_x, C_y, amt_tries = amt_tries) + amt_tries += 1 + new_x = self.acq_x(new_beta_and_l[0], new_beta_and_l[1], C_x, C_y, cur_amt_x) new_x_rd = self.round_x(new_x) print("new x rd is: " + str(new_x_rd)) - if(old_x_rd==new_x_rd): - print("We're in an infinite loop! Getting out") + if(old_x_rd==new_x_rd and amt_tries == 100): + print("We're in an infinite loop? Getting out.") + print("This likely means that " + str(new_x_rd) + " is an optimum") #problem: after ending infinite loop the value is added, even though it was already in C_x #..this makes the matrix singular. Shouldn't happen though with modified get_beta_and_l break @@ -204,6 +222,9 @@ def perform_BO(self, amt_it: int): C_x[cur_amt_x] = new_x_rd C_y[cur_amt_x] = self.perform_evaluation_at(new_x_rd) cur_amt_x += 1 + if(C_y[cur_amt_x-1]==self.f_max): + print("We've reached the specified maximum of f. Stopping BO at iteration " + str(i)) + break #ends everything if cov_matr is singular. Should this be dependant on l? Cov_matr being singular should not depend on l I think print("Checking if cov_matr is singular") if(np.linalg.det(self.get_cov_matrix(C_x, cur_amt_x)) == 0): @@ -221,6 +242,7 @@ def perform_BO(self, amt_it: int): x_ret = C_x[i] print("The Best value found in " + str(amt_it) + " iterations is " + str(y_ret) + " at " + str(x_ret)) return x_ret, y_ret + #use squared exp. kernel as covariance function k, with parameters sigma_k and l_k sigma_k = 1 l_k = 0.5 @@ -329,18 +351,19 @@ def get_l_k(self, t: int): return 0.5 #gets new beta and l if old beta and l give values that are already in C - def get_new_beta_and_l(self, cur_beta: float, cur_amt_x, cur_x, C_x, C_y): + def get_new_beta_and_l(self, cur_beta: float, cur_amt_x, cur_x, C_x, C_y, amt_tries: int=0): print("Getting new beta and l. Current beta: " + str(cur_beta) +", Current l: " + str(self.l_k)) + print("Try: " + str(amt_tries)) if(np.linalg.det(self.get_cov_matrix(C_x, cur_amt_x))==0): print("Getting new beta and l, but suddenly the cov matr is singular??") else: print("Apparently the cov_matr is not singular. Getting new beta and l") #making upper bounds dependable on current values so bounds are never too low - beta_h = cur_beta+100 - l_h = self.l_k+50 + beta_h = cur_beta+(100*2**amt_tries) + l_h = self.l_k+(50*2**amt_tries) #0 if rd(x) is in ev set C_x, constant otherwise (5) #penalty p(x) erhöhen wenn gleicher Wert rauskommt. z.B. immer +50 oder *2 oder anders exponentiell - p = lambda x: self.check_if_in_array(self.round_x(x), C_x)*50 + p = lambda x: self.check_if_in_array(self.round_x(x), C_x)*(50*2**amt_tries) #gets the x value for certain l (z[1]) and beta (z[0]) new_x = lambda z: self.acq_x(z[0], z[1], C_x, C_y, cur_amt_x) #for g: x[0] is \beta+d\beta, x[1] is l. Also d\beta = \beta+d\beta-\beta @@ -467,7 +490,7 @@ def __init__(self, data_name: str = "moons", samples: int = 500, dimension: int self.max_lv = max_lv self.max_evals = max_evals self.classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1], ["list", 0, 1]] - self.class_dim_wise_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1], ["list", 0, 1, 2], ["interval", 0, 1], ["list", 0, 1], ["list", 0, 1]] + self.class_dim_wise_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1], ["list", 0, 1, 2], ["interval", 0, 1], ["list", 0, 1], ["list", 0, 1], ["interval_int", 2, self.max_evals]] def create_data(self, name: str = "moons"): dataset = deml.datasets.make_moons(n_samples=self.samples, noise=0.15, random_state=1) @@ -510,8 +533,8 @@ def pea_classification_dimension_wise(self, params): if(len(params)<4): print("too little params for pea_classification. Returning 0.0") return 0.0 - #lambd, massl, min_lv, one_vs_others / error calc, margin(float 0-1), rebalancing (bool), use_relative_surplus (bool), - params = [float(params[0]), int(params[1]), int(params[2]), int(params[3]), float(params[4]), int(params[5]), int(params[6])] + #lambd, massl, min_lv, one_vs_others / error calc, margin(float 0-1), rebalancing (bool), use_relative_surplus (bool), max_evaluations (int) + params = [float(params[0]), int(params[1]), int(params[2]), int(params[3]), float(params[4]), int(params[5]), int(params[6]), int(params[7])] cur_data = self.data.copy() error_calculator=sparseSpACE.ErrorCalculator.ErrorCalculatorSingleDimMisclassificationGlobal() ec = None @@ -522,7 +545,7 @@ def pea_classification_dimension_wise(self, params): elif(params[3] == 1): ovo = True classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) - classification.perform_classification_dimension_wise(masslumping=params[1], lambd=params[0], minimum_level=params[2], maximum_level=self.max_lv, one_vs_others=ovo, error_calculator = ec, margin = params[4], rebalancing = params[5], use_relative_surplus = params[6], print_metrics=False) + classification.perform_classification_dimension_wise(masslumping=params[1], lambd=params[0], minimum_level=params[2], maximum_level=self.max_lv, one_vs_others=ovo, error_calculator = ec, margin = params[4], rebalancing = params[5], use_relative_surplus = params[6], max_evaluations = params[7], print_metrics=False) evaluation = classification.evaluate() print("Percentage of correct mappings",evaluation["Percentage correct"]) #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion @@ -531,12 +554,17 @@ def pea_classification_dimension_wise(self, params): #anderes dataset ausprobieren?, vllt höhere dimensionen, bis zu dim=10. Note: moons immer 2d, aber z.B. def simple_test(params): return params[0] -simple_space = [["interval_int", 3.4, 0.7], ["list", 1, 3, 4], ["interval", 2], ["list", 0, 1]] - -#OC = Optimize_Classification(data_name = "iris", dimension = 2) -#HPO = HP_Optimization(OC.pea_classification, OC.classification_space) -HPO = HP_Optimization(simple_test, simple_space) -print(HPO.hp_space) -print(HPO.round_x([2.9, 2, 1, 2])) -#print(HPO.cart_prod_hp_space(2)) +simple_space = [["interval_int", 5.4, 0.7], ["list", 1, 3, 4], ["interval", 2], ["list", 0, 1]] + +OC = Optimize_Classification(data_name = "circles", dimension = 2) +HPO = HP_Optimization(OC.pea_classification_dimension_wise, OC.class_dim_wise_space, f_max = 1) +#HPO.perform_BO(5) +HPO.perform_RO(5) + +#print(OC.pea_classification_dimension_wise([0.0, 0, 1, 0, 0.0, 0, 0, 2])) + +#HPO = HP_Optimization(simple_test, simple_space) +#print(HPO.hp_space) +#HPO.perform_GO(search_space=[[1, 1, 2, 1], [5, 4, 2, 0], [3, 3, 2, 1]]) + #HPO.perform_BO(3) From 74f46ad903376313ab5e5fe894e2ec7c4c1be81a Mon Sep 17 00:00:00 2001 From: Sonja Date: Wed, 18 Aug 2021 11:06:43 +0200 Subject: [PATCH 45/70] All opt functions now return best_x, best_x, set_x, set_y with the sets being all values used / gotten in the optimization process --- sparseSpACE/HP_Optimization.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index f089885..184d674 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -57,6 +57,8 @@ def perform_GO(self, interval_levels = 4, search_space = None): #best_time = None #storing the parameters of the best evaluation best_x = None + x_set = [] + y_set = [] #todo: split Dataset?? #original classification: classification = deml.Classification(data, split_percentage=0.8, split_evenly=True, shuffle_data=True) #original settings: classification.perform_classification(masslumping=True, lambd=0.0, minimum_level=1, maximum_level=5, print_metrics=True) @@ -66,11 +68,13 @@ def perform_GO(self, interval_levels = 4, search_space = None): #Note: lambd vor Allem sinnvoll wenn masslumping aus ist, sollte kaum unterschied machen wenn an if(search_space == None): search_space = self.cart_prod_hp_space(interval_levels) + x_set = search_space for x in search_space: #use "perform evaluation at" #cur_time = classification._time_used #print ("current time needed = " + str(cur_time)) cur_evaluation = self.perform_evaluation_at(x) + y_set.append(cur_evaluation) print("Current Evaluation: ",cur_evaluation) #if best_evaluation == None or cur_evaluation > best_evaluation or (cur_evaluation == best_evaluation and (best_time == None or best_time>cur_time)): if(best_evaluation == None or cur_evaluation>best_evaluation): @@ -83,7 +87,7 @@ def perform_GO(self, interval_levels = 4, search_space = None): print("We've reached the specified maximum of f. Stopping GO.") break print("In the end, best evaluation is " + str(best_evaluation) + " at " + str(best_x)) - return best_x, best_evaluation + return best_x, best_evaluation, x_set, y_set #+ " and time = " + str(best_time)) def cart_prod_hp_space(self, interval_levels): @@ -133,10 +137,14 @@ def cart_prod_hp_space(self, interval_levels): def perform_RO(self, amt_it: int): best_evaluation = 0 best_x = None + x_set = [] + y_set = [] for i in range (0, amt_it, 1): print("Random step " + str(i)) x = self.create_random_x() new_eval = self.perform_evaluation_at(x) + x_set.append(x) + y_set.append(new_eval) if new_eval>best_evaluation: best_x = x best_evaluation = new_eval @@ -144,7 +152,7 @@ def perform_RO(self, amt_it: int): print("We've reached the specified maximum of f. Stopping RO at iteration " + str(i)) break print("Best evaluation in " + str(amt_it) + " random steps: " + str(best_evaluation) + " at " + str(best_x)) - return best_x, best_evaluation + return best_x, best_evaluation, x_set, y_set #returns evaluation for certain Parameters on a certain data set def perform_evaluation_at(self, params): @@ -241,7 +249,7 @@ def perform_BO(self, amt_it: int): y_ret = C_y[i] x_ret = C_x[i] print("The Best value found in " + str(amt_it) + " iterations is " + str(y_ret) + " at " + str(x_ret)) - return x_ret, y_ret + return x_ret, y_ret, C_x, C_y #use squared exp. kernel as covariance function k, with parameters sigma_k and l_k sigma_k = 1 @@ -559,7 +567,7 @@ def simple_test(params): OC = Optimize_Classification(data_name = "circles", dimension = 2) HPO = HP_Optimization(OC.pea_classification_dimension_wise, OC.class_dim_wise_space, f_max = 1) #HPO.perform_BO(5) -HPO.perform_RO(5) +print(HPO.perform_GO(1)[3]) #print(OC.pea_classification_dimension_wise([0.0, 0, 1, 0, 0.0, 0, 0, 2])) From 791aecad6d1a71b7bab5e6a3606517e4c750cb93 Mon Sep 17 00:00:00 2001 From: Sonja Date: Wed, 18 Aug 2021 16:54:15 +0200 Subject: [PATCH 46/70] acq_x doesn't work anymore - the minimizing drops error and only returns smallest x possible. NO idea why --- sparseSpACE/HP_Optimization.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 184d674..fd6e741 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -200,7 +200,7 @@ def perform_BO(self, amt_it: int): #value that will be evaluated and added to the evidence set print("Getting new x:") new_x=self.acq_x(beta, l, C_x, C_y, cur_amt_x) - print("new x: " + str(new_x)) + print("acq_x gives new x = " + str(new_x)) new_x_rd = self.round_x(new_x) old_x_rd = None old_beta_and_l = [beta, l] @@ -216,9 +216,9 @@ def perform_BO(self, amt_it: int): new_x = self.acq_x(new_beta_and_l[0], new_beta_and_l[1], C_x, C_y, cur_amt_x) new_x_rd = self.round_x(new_x) print("new x rd is: " + str(new_x_rd)) - if(old_x_rd==new_x_rd and amt_tries == 100): + if(old_x_rd==new_x_rd and amt_tries == 10): print("We're in an infinite loop? Getting out.") - print("This likely means that " + str(new_x_rd) + " is an optimum") + print("This likely means that " + str(new_x_rd) + " is an optimum??") #problem: after ending infinite loop the value is added, even though it was already in C_x #..this makes the matrix singular. Shouldn't happen though with modified get_beta_and_l break @@ -351,7 +351,7 @@ def get_beta(self, t: int): a: float = 1 b: float = 1 beta = 2*math.log(t**2*2*math.pi**2/3*delta)+2*d*math.log(t**2*d*b*r*math.sqrt(math.log(4*d*a/delta))) - print(beta) + print("get_beta returns: " + str(beta)) return beta #TODO: get l_k and use it @@ -422,21 +422,30 @@ def get_bounds(self): def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): old_l=self.l_k self.l_k=l + print("get cov matrix") K_matr = self.get_cov_matrix(C_x, cur_amt_x) if(np.linalg.det(K_matr) == 0): print("Covariance Matrix is indeed singular!") #return a value that's bullshit? like [0, 0, 0, 0] (min lv is >=1) #print(K_matr) + print("calc mu") mu = lambda x: self.get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(C_y)) + print("calc sigma_sqrd") sigma_sqrd = lambda x: self.cov(x, x)-self.get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(self.get_cov_vector(C_x, x, cur_amt_x))) #takes sqrt of abs(sigma_sqrd) bc otherwise fmin gives an error - might be an overall warning sign tho + print("calc sigma") sigma = lambda x: math.sqrt(abs(sigma_sqrd(x))) + print("calc alpha") alpha = lambda x: mu(x)+(math.sqrt(beta))*sigma(x) #negates alpha bc maximum has to be found + print("calc alpha neg") alpha_neg = lambda x: -alpha(x) + print("get bounds") bounds_an = self.get_bounds() #bounds search space to vicinity of useful values + print("bounds: " + str(bounds_an)) #problem: Nelder-Mead (same as fmin) cannot handle bounds -> use L-BFGS-B for now - x0 = np.zeros(len(self.hp_space)) + x0 = np.zeros(len(self.hp_space))+2 + print("getting new x") new_x=minimize(alpha_neg, x0, method='L-BFGS-B', bounds=bounds_an).x #print("new x: " + str(new_x) + " with function value: " + str(alpha(new_x))) return new_x @@ -564,11 +573,16 @@ def simple_test(params): return params[0] simple_space = [["interval_int", 5.4, 0.7], ["list", 1, 3, 4], ["interval", 2], ["list", 0, 1]] -OC = Optimize_Classification(data_name = "circles", dimension = 2) -HPO = HP_Optimization(OC.pea_classification_dimension_wise, OC.class_dim_wise_space, f_max = 1) +OC = Optimize_Classification(data_name = "moons", dimension = 2) +HPO = HP_Optimization(OC.pea_classification, OC.classification_space, f_max = 1) #HPO.perform_BO(5) -print(HPO.perform_GO(1)[3]) - +#y_r = HPO.perform_RO(10)[3] +sol_b = HPO.perform_BO(10) +y_b = sol_b[3] +x_b = sol_b[2] +#print("y random: " + str(y_r)) +print("y bayesian: " + str(y_b)) +print("x bayesian: " + str(x_b)) #print(OC.pea_classification_dimension_wise([0.0, 0, 1, 0, 0.0, 0, 0, 2])) #HPO = HP_Optimization(simple_test, simple_space) From afa5a72f311e2fe861f5cb5c6cd3ea7f7e611c0e Mon Sep 17 00:00:00 2001 From: Sonja Date: Wed, 18 Aug 2021 17:49:23 +0200 Subject: [PATCH 47/70] improved it a little by expanding bounds --- sparseSpACE/HP_Optimization.py | 49 +++++++++++++++++----------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index fd6e741..7676e07 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -188,6 +188,7 @@ def perform_BO(self, amt_it: int): print("iteration: " + str(i)) beta = self.get_beta(i+1) l = self.get_l_k(i) + print("beta: " + str(beta) + ", cur_amt_x: " + str(cur_amt_x)) if(np.linalg.det(self.get_cov_matrix(C_x, cur_amt_x))==0): print("With the nex x value added the Covariance Matrix is singular.\nBayesian Optimization can not be continued and will be ended here, at the start of iteration " + str(i)) break @@ -388,10 +389,10 @@ def get_new_beta_and_l(self, cur_beta: float, cur_amt_x, cur_x, C_x, C_y, amt_tr def get_bounds(self): res = [] for i in range(0, len(self.hp_space)): - new = [1, 1] + new = [0, 1] #should one be able to enter just one value for "list"?? if(len(self.hp_space[i])<2): - print("please enter at least 1 value for hp_space list. Using default values [1, 1]") + print("please enter at least 1 value for hp_space list. Using default values [0, 1]") elif(self.hp_space[i][0] == "list"): max = self.hp_space[i][1] min = max @@ -400,21 +401,21 @@ def get_bounds(self): min = self.hp_space[i][j] if(self.hp_space[i][j]>max): max = self.hp_space[i][j] - new = [min, max] + new = [min-5, max+5] elif(len(self.hp_space[i])<3): print("please enter at least 2 values for hp_space interval. Using the one value given") - new = [self.hp_space[i][1], self.hp_space[i][1]] + new = [self.hp_space[i][1]-5, self.hp_space[i][1]+5] elif(self.hp_space[i][0] == "interval"): new = self.hp_space[i].copy() new.remove("interval") - new = [new[0], new[1]] + new = [new[0]-5, new[1]+5] elif(self.hp_space[i][0] == "interval_int"): new = self.hp_space[i].copy() new.remove("interval_int") - new = [new[0], new[1]] + new = [new[0]-5, new[1]+5] else: print("please enter a valid hp_space. Using first value given") - new = [self.hp_space[i][1], self.hp_space[i][1]] + new = [self.hp_space[i][1]-5, self.hp_space[i][1]+5] res.append(new) return res @@ -422,25 +423,18 @@ def get_bounds(self): def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): old_l=self.l_k self.l_k=l - print("get cov matrix") K_matr = self.get_cov_matrix(C_x, cur_amt_x) if(np.linalg.det(K_matr) == 0): print("Covariance Matrix is indeed singular!") #return a value that's bullshit? like [0, 0, 0, 0] (min lv is >=1) #print(K_matr) - print("calc mu") mu = lambda x: self.get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(C_y)) - print("calc sigma_sqrd") sigma_sqrd = lambda x: self.cov(x, x)-self.get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(self.get_cov_vector(C_x, x, cur_amt_x))) #takes sqrt of abs(sigma_sqrd) bc otherwise fmin gives an error - might be an overall warning sign tho - print("calc sigma") sigma = lambda x: math.sqrt(abs(sigma_sqrd(x))) - print("calc alpha") alpha = lambda x: mu(x)+(math.sqrt(beta))*sigma(x) #negates alpha bc maximum has to be found - print("calc alpha neg") alpha_neg = lambda x: -alpha(x) - print("get bounds") bounds_an = self.get_bounds() #bounds search space to vicinity of useful values print("bounds: " + str(bounds_an)) #problem: Nelder-Mead (same as fmin) cannot handle bounds -> use L-BFGS-B for now @@ -448,6 +442,7 @@ def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): print("getting new x") new_x=minimize(alpha_neg, x0, method='L-BFGS-B', bounds=bounds_an).x #print("new x: " + str(new_x) + " with function value: " + str(alpha(new_x))) + print("acq_x returns: " + str(new_x)) return new_x #needs search space @@ -527,7 +522,7 @@ def create_data(self, name: str = "moons"): dataset = deml.datasets.load_breast_cancer(return_X_y=True) # hint: try only with max_level <= 4 elif(name == "wine"): dataset = deml.datasets.load_wine(return_X_y=True) - else: + elif(name != "moons"): print("This is not a valid name for a dataset, using moons") data = deml.DataSet(dataset, name="data_"+name) return data @@ -573,20 +568,26 @@ def simple_test(params): return params[0] simple_space = [["interval_int", 5.4, 0.7], ["list", 1, 3, 4], ["interval", 2], ["list", 0, 1]] -OC = Optimize_Classification(data_name = "moons", dimension = 2) -HPO = HP_Optimization(OC.pea_classification, OC.classification_space, f_max = 1) +OC = Optimize_Classification(data_name = "circles", dimension = 2) +HPO = HP_Optimization(OC.pea_classification_dimension_wise, OC.class_dim_wise_space, f_max = 1) #HPO.perform_BO(5) #y_r = HPO.perform_RO(10)[3] -sol_b = HPO.perform_BO(10) -y_b = sol_b[3] -x_b = sol_b[2] +#sol_b = HPO.perform_BO(6) +#y_b = sol_b[3] +#x_b = sol_b[2] #print("y random: " + str(y_r)) -print("y bayesian: " + str(y_b)) -print("x bayesian: " + str(x_b)) +#print("y bayesian: " + str(y_b)) +#print("x bayesian: " + str(x_b)) #print(OC.pea_classification_dimension_wise([0.0, 0, 1, 0, 0.0, 0, 0, 2])) #HPO = HP_Optimization(simple_test, simple_space) #print(HPO.hp_space) #HPO.perform_GO(search_space=[[1, 1, 2, 1], [5, 4, 2, 0], [3, 3, 2, 1]]) - -#HPO.perform_BO(3) +HPO.perform_BO(5) +art_C_y = np.array([0.96, 0.94, 0.96, 0.94, 0.97, 0.]) +art_C_x = np.array([[0.3273086, 1., 1., 1.], [0.8158708, 0., 1., 0.], [0.47385773, 1., 1., 1.], [0.80103041, 0., 1., 0.], [0.48143013, 0., 1., 1.], [0., 0., 0., 0.]]) +art_beta = 19.99433798961168 +art_l = 0.5 +art_cur_amt_x = 5 +#test_acq_x = HPO.acq_x(art_beta, art_l, art_C_x, art_C_y, art_cur_amt_x) +#print("test_acq_x: " + str(test_acq_x)) From 6a2aa3fed95d8c70b47f43b0524e71e3b61f632a Mon Sep 17 00:00:00 2001 From: Sonja Date: Wed, 18 Aug 2021 19:28:34 +0200 Subject: [PATCH 48/70] I have no idea what's happening anymore --- sparseSpACE/HP_Optimization.py | 45 ++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 7676e07..5d1ec0e 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -12,11 +12,15 @@ #the function on which HP should be Optimized and search space for the HPs #space is in form [["interval", , ], ["list", , , ,...], ....] class HP_Optimization: - def __init__(self, function, hp_space, f_max = None): + def __init__(self, function, hp_space, f_max = None, r: float = 3, delta: float = 0.1, a: float = 1., b: float = 1.): self.function = function self.hp_space = hp_space self.check_hp_space() self.f_max = f_max + self.r = r + self.delta = delta + self.a = a + self.b = b def check_hp_space(self): if(len(self.hp_space)<1): @@ -217,7 +221,7 @@ def perform_BO(self, amt_it: int): new_x = self.acq_x(new_beta_and_l[0], new_beta_and_l[1], C_x, C_y, cur_amt_x) new_x_rd = self.round_x(new_x) print("new x rd is: " + str(new_x_rd)) - if(old_x_rd==new_x_rd and amt_tries == 10): + if(old_x_rd==new_x_rd and amt_tries == 3): print("We're in an infinite loop? Getting out.") print("This likely means that " + str(new_x_rd) + " is an optimum??") #problem: after ending infinite loop the value is added, even though it was already in C_x @@ -231,9 +235,9 @@ def perform_BO(self, amt_it: int): C_x[cur_amt_x] = new_x_rd C_y[cur_amt_x] = self.perform_evaluation_at(new_x_rd) cur_amt_x += 1 - if(C_y[cur_amt_x-1]==self.f_max): - print("We've reached the specified maximum of f. Stopping BO at iteration " + str(i)) - break + if(C_y[cur_amt_x-1]>=self.f_max): + print("We've reached the specified maximum of f. Stopping BO at iteration " + str(i)) + break #ends everything if cov_matr is singular. Should this be dependant on l? Cov_matr being singular should not depend on l I think print("Checking if cov_matr is singular") if(np.linalg.det(self.get_cov_matrix(C_x, cur_amt_x)) == 0): @@ -346,11 +350,11 @@ def check_if_in_array(self, x, array): #calculates and returns current beta. t is current iteration def get_beta(self, t: int): - r: float = 1.5 - d: int = 4 - delta: float = 0.1 - a: float = 1 - b: float = 1 + r: float = self.r + d: int = len(self.hp_space) + delta: float = self.delta + a: float = self.a + b: float = self.b beta = 2*math.log(t**2*2*math.pi**2/3*delta)+2*d*math.log(t**2*d*b*r*math.sqrt(math.log(4*d*a/delta))) print("get_beta returns: " + str(beta)) return beta @@ -401,21 +405,21 @@ def get_bounds(self): min = self.hp_space[i][j] if(self.hp_space[i][j]>max): max = self.hp_space[i][j] - new = [min-5, max+5] + new = [min-2, max+2] elif(len(self.hp_space[i])<3): print("please enter at least 2 values for hp_space interval. Using the one value given") - new = [self.hp_space[i][1]-5, self.hp_space[i][1]+5] + new = [self.hp_space[i][1]-2, self.hp_space[i][1]+2] elif(self.hp_space[i][0] == "interval"): new = self.hp_space[i].copy() new.remove("interval") - new = [new[0]-5, new[1]+5] + new = [new[0]-2, new[1]+2] elif(self.hp_space[i][0] == "interval_int"): new = self.hp_space[i].copy() new.remove("interval_int") - new = [new[0]-5, new[1]+5] + new = [new[0]-2, new[1]+2] else: print("please enter a valid hp_space. Using first value given") - new = [self.hp_space[i][1]-5, self.hp_space[i][1]+5] + new = [self.hp_space[i][1]-2, self.hp_space[i][1]+2] res.append(new) return res @@ -436,10 +440,8 @@ def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): #negates alpha bc maximum has to be found alpha_neg = lambda x: -alpha(x) bounds_an = self.get_bounds() #bounds search space to vicinity of useful values - print("bounds: " + str(bounds_an)) #problem: Nelder-Mead (same as fmin) cannot handle bounds -> use L-BFGS-B for now x0 = np.zeros(len(self.hp_space))+2 - print("getting new x") new_x=minimize(alpha_neg, x0, method='L-BFGS-B', bounds=bounds_an).x #print("new x: " + str(new_x) + " with function value: " + str(alpha(new_x))) print("acq_x returns: " + str(new_x)) @@ -566,10 +568,11 @@ def pea_classification_dimension_wise(self, params): #anderes dataset ausprobieren?, vllt höhere dimensionen, bis zu dim=10. Note: moons immer 2d, aber z.B. def simple_test(params): return params[0] -simple_space = [["interval_int", 5.4, 0.7], ["list", 1, 3, 4], ["interval", 2], ["list", 0, 1]] +simple_space = [["list", 2.4, 0.7, 3.]] -OC = Optimize_Classification(data_name = "circles", dimension = 2) -HPO = HP_Optimization(OC.pea_classification_dimension_wise, OC.class_dim_wise_space, f_max = 1) +OC = Optimize_Classification(data_name = "moons", dimension = 2) +#HPO = HP_Optimization(OC.pea_classification, OC.classification_space, f_max = 1, r = 2) +HPO = HP_Optimization(simple_test, simple_space, f_max = 100, r = 2) #HPO.perform_BO(5) #y_r = HPO.perform_RO(10)[3] #sol_b = HPO.perform_BO(6) @@ -583,7 +586,7 @@ def simple_test(params): #HPO = HP_Optimization(simple_test, simple_space) #print(HPO.hp_space) #HPO.perform_GO(search_space=[[1, 1, 2, 1], [5, 4, 2, 0], [3, 3, 2, 1]]) -HPO.perform_BO(5) +HPO.perform_BO(3) art_C_y = np.array([0.96, 0.94, 0.96, 0.94, 0.97, 0.]) art_C_x = np.array([[0.3273086, 1., 1., 1.], [0.8158708, 0., 1., 0.], [0.47385773, 1., 1., 1.], [0.80103041, 0., 1., 0.], [0.48143013, 0., 1., 1.], [0., 0., 0., 0.]]) art_beta = 19.99433798961168 From cb7b4043f5916cc143887e8aa0999614e8ee9ed7 Mon Sep 17 00:00:00 2001 From: Sonja Date: Thu, 19 Aug 2021 10:51:55 +0200 Subject: [PATCH 49/70] - Added workaround: A random x is now used if no beta and l can be found that give a new x. - Made sure min() has a good x0 that is at least in the bounds (both for acq_x and get_new_beta_and_l) -> seems to have improved speed greatly --- sparseSpACE/HP_Optimization.py | 39 +++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 5d1ec0e..59a576d 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -221,11 +221,14 @@ def perform_BO(self, amt_it: int): new_x = self.acq_x(new_beta_and_l[0], new_beta_and_l[1], C_x, C_y, cur_amt_x) new_x_rd = self.round_x(new_x) print("new x rd is: " + str(new_x_rd)) - if(old_x_rd==new_x_rd and amt_tries == 3): - print("We're in an infinite loop? Getting out.") - print("This likely means that " + str(new_x_rd) + " is an optimum??") + if(old_x_rd==new_x_rd and amt_tries >= 3): + print("It seems a beta and l that give a new x cannot be found. Using random new x and then getting out.") + print("If a random x that's not in C_x can not be found in 10 tries it is assumed that all possible x have been used already and BO is stopped") #problem: after ending infinite loop the value is added, even though it was already in C_x #..this makes the matrix singular. Shouldn't happen though with modified get_beta_and_l + while(self.check_if_in_array(new_x_rd, C_x) and amt_tries < 13): + new_x_rd = self.create_random_x() + amt_tries += 1 break print("out of loop") if(old_x_rd==new_x_rd): @@ -321,7 +324,6 @@ def create_random_x(self): sel = self.hp_space[i].copy() sel.remove("list") new_x = random.choice(sel) - print("hp_space[" + str(i) + "] is now " + str(self.hp_space[i])) elif (len(self.hp_space[i])<3): print("Please enter at least 2 values for hp_space interval! Using the 1 value given") new_x = self.hp_space[i][1] @@ -385,13 +387,14 @@ def get_new_beta_and_l(self, cur_beta: float, cur_amt_x, cur_x, C_x, C_y, amt_tr print("About to minimize g(...)") #due to numerical inaccuracies a matrix might become singular with new beta and l #(even though it wouldn't be) mathematically - how to deal with that?? - result = minimize(g, [1, 1], method='L-BFGS-B', bounds=bounds_g).x + result = minimize(g, [cur_beta, self.l_k], method='L-BFGS-B', bounds=bounds_g).x print("New beta: " + str(result[0]) + ", New l: " + str(result[1])) #result is in the form [new beta, new l] return result - def get_bounds(self): - res = [] + def get_bounds_and_x0(self): + bounds = [] + x0 = np.zeros(len(self.hp_space)) for i in range(0, len(self.hp_space)): new = [0, 1] #should one be able to enter just one value for "list"?? @@ -420,8 +423,9 @@ def get_bounds(self): else: print("please enter a valid hp_space. Using first value given") new = [self.hp_space[i][1]-2, self.hp_space[i][1]+2] - res.append(new) - return res + bounds.append(new) + x0[i] = new[0] + return bounds, x0 #acquire new x for l, beta, C_x, cur_amt_x, using GP-UCB def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): @@ -436,12 +440,13 @@ def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): sigma_sqrd = lambda x: self.cov(x, x)-self.get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(self.get_cov_vector(C_x, x, cur_amt_x))) #takes sqrt of abs(sigma_sqrd) bc otherwise fmin gives an error - might be an overall warning sign tho sigma = lambda x: math.sqrt(abs(sigma_sqrd(x))) - alpha = lambda x: mu(x)+(math.sqrt(beta))*sigma(x) + alpha = lambda x: mu(x)+(math.sqrt(abs(beta)))*sigma(x) #negates alpha bc maximum has to be found alpha_neg = lambda x: -alpha(x) - bounds_an = self.get_bounds() #bounds search space to vicinity of useful values + bounds_and_x0 = self.get_bounds_and_x0() #bounds search space to vicinity of useful values + bounds_an = bounds_and_x0[0] #problem: Nelder-Mead (same as fmin) cannot handle bounds -> use L-BFGS-B for now - x0 = np.zeros(len(self.hp_space))+2 + x0 = bounds_and_x0[1] new_x=minimize(alpha_neg, x0, method='L-BFGS-B', bounds=bounds_an).x #print("new x: " + str(new_x) + " with function value: " + str(alpha(new_x))) print("acq_x returns: " + str(new_x)) @@ -570,10 +575,10 @@ def simple_test(params): return params[0] simple_space = [["list", 2.4, 0.7, 3.]] -OC = Optimize_Classification(data_name = "moons", dimension = 2) -#HPO = HP_Optimization(OC.pea_classification, OC.classification_space, f_max = 1, r = 2) -HPO = HP_Optimization(simple_test, simple_space, f_max = 100, r = 2) -#HPO.perform_BO(5) +OC = Optimize_Classification(data_name = "circles", dimension = 2) +HPO = HP_Optimization(OC.pea_classification, OC.classification_space, f_max = 1, r = 2) +#HPO = HP_Optimization(simple_test, simple_space, f_max = 100, r = 2) +HPO.perform_BO(10) #y_r = HPO.perform_RO(10)[3] #sol_b = HPO.perform_BO(6) #y_b = sol_b[3] @@ -586,7 +591,7 @@ def simple_test(params): #HPO = HP_Optimization(simple_test, simple_space) #print(HPO.hp_space) #HPO.perform_GO(search_space=[[1, 1, 2, 1], [5, 4, 2, 0], [3, 3, 2, 1]]) -HPO.perform_BO(3) +#HPO.perform_BO(3) art_C_y = np.array([0.96, 0.94, 0.96, 0.94, 0.97, 0.]) art_C_x = np.array([[0.3273086, 1., 1., 1.], [0.8158708, 0., 1., 0.], [0.47385773, 1., 1., 1.], [0.80103041, 0., 1., 0.], [0.48143013, 0., 1., 1.], [0., 0., 0., 0.]]) art_beta = 19.99433798961168 From 915734f1af640a23dc138dee9e792132d90ecc60 Mon Sep 17 00:00:00 2001 From: Sonja Date: Thu, 19 Aug 2021 11:31:56 +0200 Subject: [PATCH 50/70] Fixed mistake, now BO actually stops when no new x value is found by sampling random x --- sparseSpACE/HP_Optimization.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 59a576d..e42e804 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -231,14 +231,14 @@ def perform_BO(self, amt_it: int): amt_tries += 1 break print("out of loop") - if(old_x_rd==new_x_rd): - print("There was an infinite loop when getting new x. Ending BO at iteration " + str(i)) + if(old_x_rd==new_x_rd or self.check_if_in_array(new_x_rd, C_x)): + print("There was an infinite loop when getting new x and / or no new x has been found. Ending BO at iteration " + str(i)) break print("adding " + str(new_x_rd)) C_x[cur_amt_x] = new_x_rd C_y[cur_amt_x] = self.perform_evaluation_at(new_x_rd) cur_amt_x += 1 - if(C_y[cur_amt_x-1]>=self.f_max): + if(C_y[cur_amt_x-1]==self.f_max): print("We've reached the specified maximum of f. Stopping BO at iteration " + str(i)) break #ends everything if cov_matr is singular. Should this be dependant on l? Cov_matr being singular should not depend on l I think @@ -576,7 +576,7 @@ def simple_test(params): simple_space = [["list", 2.4, 0.7, 3.]] OC = Optimize_Classification(data_name = "circles", dimension = 2) -HPO = HP_Optimization(OC.pea_classification, OC.classification_space, f_max = 1, r = 2) +HPO = HP_Optimization(simple_test, simple_space, r = 2) #HPO = HP_Optimization(simple_test, simple_space, f_max = 100, r = 2) HPO.perform_BO(10) #y_r = HPO.perform_RO(10)[3] From c160fd0ae709bcaae5eb31a90a216bda5b0bca98 Mon Sep 17 00:00:00 2001 From: Sonja Date: Thu, 19 Aug 2021 15:59:17 +0200 Subject: [PATCH 51/70] BO does practically RO if BO doesn't work --- sparseSpACE/HP_Optimization.py | 68 +++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 26 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index e42e804..89a3870 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -188,6 +188,8 @@ def perform_BO(self, amt_it: int): C_y=C[1] print("C_y: " + str(C_y)) #print("Evidence Set: \n" + str(C)) + #For easier checking in what iterations random x has been used + used_random_x = [] for i in range (0, amt_it, 1): print("iteration: " + str(i)) beta = self.get_beta(i+1) @@ -222,15 +224,20 @@ def perform_BO(self, amt_it: int): new_x_rd = self.round_x(new_x) print("new x rd is: " + str(new_x_rd)) if(old_x_rd==new_x_rd and amt_tries >= 3): - print("It seems a beta and l that give a new x cannot be found. Using random new x and then getting out.") - print("If a random x that's not in C_x can not be found in 10 tries it is assumed that all possible x have been used already and BO is stopped") - #problem: after ending infinite loop the value is added, even though it was already in C_x - #..this makes the matrix singular. Shouldn't happen though with modified get_beta_and_l - while(self.check_if_in_array(new_x_rd, C_x) and amt_tries < 13): - new_x_rd = self.create_random_x() - amt_tries += 1 - break - print("out of loop") + print("It seems a beta and l that give a new x cannot be found. Trying beta = 0 and l = 0.5") + new_x = self.acq_x(0, 0.5, C_x, C_y, cur_amt_x) + new_x_rd = self.round_x(new_x) + if(self.check_if_in_array(new_x_rd, C_x)): + print("Beta = 0 didn't help. Using random new x and then getting out.") + print("If a random x that's not in C_x can not be found in 10 tries it is assumed that all possible x have been used already and BO is stopped") + used_random_x.append(i) + while(self.check_if_in_array(new_x_rd, C_x) and amt_tries < 13): + new_x_rd = self.create_random_x() + amt_tries += 1 + break + else: + print("Beta = 0 DID help!") + print("out of getting beta and l loop") if(old_x_rd==new_x_rd or self.check_if_in_array(new_x_rd, C_x)): print("There was an infinite loop when getting new x and / or no new x has been found. Ending BO at iteration " + str(i)) break @@ -257,6 +264,9 @@ def perform_BO(self, amt_it: int): y_ret = C_y[i] x_ret = C_x[i] print("The Best value found in " + str(amt_it) + " iterations is " + str(y_ret) + " at " + str(x_ret)) + print("Random x has been used in iterations " + str(used_random_x)) + print("C_y: " + str(C_y)) + print("C_x: " + str(C_x)) return x_ret, y_ret, C_x, C_y #use squared exp. kernel as covariance function k, with parameters sigma_k and l_k @@ -286,14 +296,14 @@ def get_cov_matrix(self, C_x, cur_length: int): return K #returns the vector k for a certain input new_x - def get_cov_vector(self, ev_x, new_x, cur_length: int): + def get_cov_vector(self, C_x, new_x, cur_length: int): #somehow the rest of the function would still work even if x is wrong size, but no idea why / what it does. Better to check - if(len(ev_x[0]) != len(new_x)): + if(len(C_x[0]) != len(new_x)): print("Input x for cov_vector has the wrong size") return None - k_vec = np.zeros(len(ev_x)) + k_vec = np.zeros(len(C_x)) for i in range (0, cur_length, 1): - k_vec[i] = self.cov(ev_x[i], new_x) + k_vec[i] = self.cov(C_x[i], new_x) return k_vec #creates evidence set using random values @@ -378,7 +388,7 @@ def get_new_beta_and_l(self, cur_beta: float, cur_amt_x, cur_x, C_x, C_y, amt_tr l_h = self.l_k+(50*2**amt_tries) #0 if rd(x) is in ev set C_x, constant otherwise (5) #penalty p(x) erhöhen wenn gleicher Wert rauskommt. z.B. immer +50 oder *2 oder anders exponentiell - p = lambda x: self.check_if_in_array(self.round_x(x), C_x)*(50*2**amt_tries) + p = lambda x: self.check_if_in_array(self.round_x(x), C_x)*(200*2**amt_tries) #gets the x value for certain l (z[1]) and beta (z[0]) new_x = lambda z: self.acq_x(z[0], z[1], C_x, C_y, cur_amt_x) #for g: x[0] is \beta+d\beta, x[1] is l. Also d\beta = \beta+d\beta-\beta @@ -408,28 +418,28 @@ def get_bounds_and_x0(self): min = self.hp_space[i][j] if(self.hp_space[i][j]>max): max = self.hp_space[i][j] - new = [min-2, max+2] + new = [min-1, max+1] elif(len(self.hp_space[i])<3): print("please enter at least 2 values for hp_space interval. Using the one value given") - new = [self.hp_space[i][1]-2, self.hp_space[i][1]+2] + new = [self.hp_space[i][1]-1, self.hp_space[i][1]+1] elif(self.hp_space[i][0] == "interval"): new = self.hp_space[i].copy() new.remove("interval") - new = [new[0]-2, new[1]+2] + new = [new[0]-1, new[1]+1] elif(self.hp_space[i][0] == "interval_int"): new = self.hp_space[i].copy() new.remove("interval_int") - new = [new[0]-2, new[1]+2] + new = [new[0]-1, new[1]+1] else: print("please enter a valid hp_space. Using first value given") - new = [self.hp_space[i][1]-2, self.hp_space[i][1]+2] + new = [self.hp_space[i][1]-1, self.hp_space[i][1]+1] bounds.append(new) x0[i] = new[0] return bounds, x0 #acquire new x for l, beta, C_x, cur_amt_x, using GP-UCB def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): - old_l=self.l_k + #old_l=self.l_k self.l_k=l K_matr = self.get_cov_matrix(C_x, cur_amt_x) if(np.linalg.det(K_matr) == 0): @@ -441,6 +451,8 @@ def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): #takes sqrt of abs(sigma_sqrd) bc otherwise fmin gives an error - might be an overall warning sign tho sigma = lambda x: math.sqrt(abs(sigma_sqrd(x))) alpha = lambda x: mu(x)+(math.sqrt(abs(beta)))*sigma(x) + #trying out stuff: + #alpha = lambda x: mu(x) #negates alpha bc maximum has to be found alpha_neg = lambda x: -alpha(x) bounds_and_x0 = self.get_bounds_and_x0() #bounds search space to vicinity of useful values @@ -449,7 +461,10 @@ def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): x0 = bounds_and_x0[1] new_x=minimize(alpha_neg, x0, method='L-BFGS-B', bounds=bounds_an).x #print("new x: " + str(new_x) + " with function value: " + str(alpha(new_x))) - print("acq_x returns: " + str(new_x)) + print("acq_x returns: " + str(new_x) + " with a_neg(new_x) = " + str(alpha_neg(new_x))) + print("mu(new_x) = " + str(mu(new_x)) + " , root(beta) = " + str(math.sqrt(beta)) + " , sigma(new_x) = " + str(sigma(new_x))) + print("sigma_sqrd(new_x) = " + str(sigma_sqrd(new_x))) + print("k_vec(x) = " + str(self.get_cov_vector(C_x, new_x, cur_amt_x))) return new_x #needs search space @@ -573,12 +588,13 @@ def pea_classification_dimension_wise(self, params): #anderes dataset ausprobieren?, vllt höhere dimensionen, bis zu dim=10. Note: moons immer 2d, aber z.B. def simple_test(params): return params[0] -simple_space = [["list", 2.4, 0.7, 3.]] +simple_space = [["interval", 20.4, 0.]] -OC = Optimize_Classification(data_name = "circles", dimension = 2) -HPO = HP_Optimization(simple_test, simple_space, r = 2) -#HPO = HP_Optimization(simple_test, simple_space, f_max = 100, r = 2) -HPO.perform_BO(10) +OC = Optimize_Classification(data_name = "moons", dimension = 2) +#HPO = HP_Optimization(OC.pea_classification, OC.classification_space, r = 1.5) +HPO = HP_Optimization(OC.pea_classification_dimension_wise, OC.class_dim_wise_space, r = 1.5) +#HPO = HP_Optimization(simple_test, simple_space, f_max = 100, r = 21) +HPO.perform_BO(7) #y_r = HPO.perform_RO(10)[3] #sol_b = HPO.perform_BO(6) #y_b = sol_b[3] From 0269d73e9f5a544e3f4e0411ed2bee16f2136d65 Mon Sep 17 00:00:00 2001 From: Sonja Date: Thu, 2 Sep 2021 13:09:04 +0200 Subject: [PATCH 52/70] Users now input parameters for calculation of beta when calling BO, not when creating the hpo object --- sparseSpACE/HP_Optimization.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 89a3870..4836e1b 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -12,15 +12,15 @@ #the function on which HP should be Optimized and search space for the HPs #space is in form [["interval", , ], ["list", , , ,...], ....] class HP_Optimization: - def __init__(self, function, hp_space, f_max = None, r: float = 3, delta: float = 0.1, a: float = 1., b: float = 1.): + def __init__(self, function, hp_space, f_max = None): self.function = function self.hp_space = hp_space self.check_hp_space() self.f_max = f_max - self.r = r - self.delta = delta - self.a = a - self.b = b + self.r = 3.0 + self.delta = 0.1 + self.a = 1.0 + self.b = 1.0 def check_hp_space(self): if(len(self.hp_space)<1): @@ -175,7 +175,11 @@ def perform_evaluation_at(self, params): #performs Bayesian Optimization #Note: HP Input of classification is (lambd:float, massl:bool, min_lv:int <4, one_vs_others:bool) - def perform_BO(self, amt_it: int): + def perform_BO(self, amt_it: int, r: float = 3.0, delta: float = 0.1, a: float = 1.0, b: float = 1.0): + self.r = r + self.delta = delta + self.a = a + self.b = b #notes how many x values currently are in the evidence set - starts with amt_HP+1 amt_HP = len(self.hp_space) cur_amt_x: int = amt_HP+1 @@ -590,11 +594,11 @@ def simple_test(params): return params[0] simple_space = [["interval", 20.4, 0.]] -OC = Optimize_Classification(data_name = "moons", dimension = 2) +OC = Optimize_Classification(data_name = "iris", dimension = 2) #HPO = HP_Optimization(OC.pea_classification, OC.classification_space, r = 1.5) -HPO = HP_Optimization(OC.pea_classification_dimension_wise, OC.class_dim_wise_space, r = 1.5) +HPO = HP_Optimization(OC.pea_classification_dimension_wise, OC.class_dim_wise_space) #HPO = HP_Optimization(simple_test, simple_space, f_max = 100, r = 21) -HPO.perform_BO(7) +HPO.perform_BO(3, r = 4) #y_r = HPO.perform_RO(10)[3] #sol_b = HPO.perform_BO(6) #y_b = sol_b[3] From 2d4504af79cf5488f38c8b720858eca9c616bbaf Mon Sep 17 00:00:00 2001 From: Sonja Date: Fri, 3 Sep 2021 09:41:58 +0200 Subject: [PATCH 53/70] users can now specify data_set themselves, not just through name --- sparseSpACE/HP_Optimization.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 4836e1b..f95278c 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -520,11 +520,14 @@ def round_x(self, x): return new_x_rd class Optimize_Classification: - def __init__(self, data_name: str = "moons", samples: int = 500, dimension: int = 2, labels: int = 6, max_lv: int = 3, max_evals: int = 256): + def __init__(self, data_name: str = "moons", data = None, samples: int = 500, dimension: int = 2, labels: int = 6, max_lv: int = 3, max_evals: int = 256): self.samples = samples self.dimension = dimension self.labels = labels - self.data = self.create_data(data_name) + if(data == None): + self.data = self.create_data(data_name) + else: + self.data = data self.max_lv = max_lv self.max_evals = max_evals self.classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1], ["list", 0, 1]] @@ -594,7 +597,9 @@ def simple_test(params): return params[0] simple_space = [["interval", 20.4, 0.]] -OC = Optimize_Classification(data_name = "iris", dimension = 2) +dataset_iris = deml.datasets.load_iris(return_X_y=True) +iris_data = deml.DataSet(dataset_iris, name="data_iris") +OC = Optimize_Classification(data= iris_data, dimension = 2) #HPO = HP_Optimization(OC.pea_classification, OC.classification_space, r = 1.5) HPO = HP_Optimization(OC.pea_classification_dimension_wise, OC.class_dim_wise_space) #HPO = HP_Optimization(simple_test, simple_space, f_max = 100, r = 21) From bdfbfcccb8dd4645ca00b67f54d71c30892a1988 Mon Sep 17 00:00:00 2001 From: Sonja Date: Fri, 3 Sep 2021 14:54:15 +0200 Subject: [PATCH 54/70] get_bounds now uses an offset for easier change of how tight the bounds should be --- sparseSpACE/HP_Optimization.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index f95278c..efcf5dc 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -408,6 +408,7 @@ def get_new_beta_and_l(self, cur_beta: float, cur_amt_x, cur_x, C_x, C_y, amt_tr def get_bounds_and_x0(self): bounds = [] + offset = 0 x0 = np.zeros(len(self.hp_space)) for i in range(0, len(self.hp_space)): new = [0, 1] @@ -422,21 +423,21 @@ def get_bounds_and_x0(self): min = self.hp_space[i][j] if(self.hp_space[i][j]>max): max = self.hp_space[i][j] - new = [min-1, max+1] + new = [min+offset, max-offset] elif(len(self.hp_space[i])<3): print("please enter at least 2 values for hp_space interval. Using the one value given") - new = [self.hp_space[i][1]-1, self.hp_space[i][1]+1] + new = [self.hp_space[i][1]-offset, self.hp_space[i][1]+offset] elif(self.hp_space[i][0] == "interval"): new = self.hp_space[i].copy() new.remove("interval") - new = [new[0]-1, new[1]+1] + new = [new[0]-offset, new[1]+offset] elif(self.hp_space[i][0] == "interval_int"): new = self.hp_space[i].copy() new.remove("interval_int") - new = [new[0]-1, new[1]+1] + new = [new[0]-offset, new[1]+offset] else: print("please enter a valid hp_space. Using first value given") - new = [self.hp_space[i][1]-1, self.hp_space[i][1]+1] + new = [self.hp_space[i][1]-offset, self.hp_space[i][1]+offset] bounds.append(new) x0[i] = new[0] return bounds, x0 @@ -599,11 +600,11 @@ def simple_test(params): dataset_iris = deml.datasets.load_iris(return_X_y=True) iris_data = deml.DataSet(dataset_iris, name="data_iris") -OC = Optimize_Classification(data= iris_data, dimension = 2) +OC = Optimize_Classification() #HPO = HP_Optimization(OC.pea_classification, OC.classification_space, r = 1.5) -HPO = HP_Optimization(OC.pea_classification_dimension_wise, OC.class_dim_wise_space) +HPO = HP_Optimization(OC.pea_classification, OC.classification_space) #HPO = HP_Optimization(simple_test, simple_space, f_max = 100, r = 21) -HPO.perform_BO(3, r = 4) +HPO.perform_BO(7) #y_r = HPO.perform_RO(10)[3] #sol_b = HPO.perform_BO(6) #y_b = sol_b[3] @@ -616,11 +617,4 @@ def simple_test(params): #HPO = HP_Optimization(simple_test, simple_space) #print(HPO.hp_space) #HPO.perform_GO(search_space=[[1, 1, 2, 1], [5, 4, 2, 0], [3, 3, 2, 1]]) -#HPO.perform_BO(3) -art_C_y = np.array([0.96, 0.94, 0.96, 0.94, 0.97, 0.]) -art_C_x = np.array([[0.3273086, 1., 1., 1.], [0.8158708, 0., 1., 0.], [0.47385773, 1., 1., 1.], [0.80103041, 0., 1., 0.], [0.48143013, 0., 1., 1.], [0., 0., 0., 0.]]) -art_beta = 19.99433798961168 -art_l = 0.5 -art_cur_amt_x = 5 -#test_acq_x = HPO.acq_x(art_beta, art_l, art_C_x, art_C_y, art_cur_amt_x) -#print("test_acq_x: " + str(test_acq_x)) + From fb0217a4099a2eddf48aeaab61d66fcad2dd23f1 Mon Sep 17 00:00:00 2001 From: Sonja Date: Fri, 3 Sep 2021 15:03:50 +0200 Subject: [PATCH 55/70] fixed a mistake with the offset thing --- sparseSpACE/HP_Optimization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index efcf5dc..23c27b8 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -423,7 +423,7 @@ def get_bounds_and_x0(self): min = self.hp_space[i][j] if(self.hp_space[i][j]>max): max = self.hp_space[i][j] - new = [min+offset, max-offset] + new = [min-offset, max+offset] elif(len(self.hp_space[i])<3): print("please enter at least 2 values for hp_space interval. Using the one value given") new = [self.hp_space[i][1]-offset, self.hp_space[i][1]+offset] @@ -604,7 +604,7 @@ def simple_test(params): #HPO = HP_Optimization(OC.pea_classification, OC.classification_space, r = 1.5) HPO = HP_Optimization(OC.pea_classification, OC.classification_space) #HPO = HP_Optimization(simple_test, simple_space, f_max = 100, r = 21) -HPO.perform_BO(7) +HPO.perform_BO(5) #y_r = HPO.perform_RO(10)[3] #sol_b = HPO.perform_BO(6) #y_b = sol_b[3] From 9522078056a1588b76252f41e7741f5e615c6093 Mon Sep 17 00:00:00 2001 From: Sonja Date: Sat, 4 Sep 2021 08:51:59 +0200 Subject: [PATCH 56/70] added plt --- sparseSpACE/HP_Optimization.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 23c27b8..e0b7a06 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -7,6 +7,7 @@ import itertools from scipy.optimize import fmin from scipy.optimize import minimize +import matplotlib.pyplot as plt #class HP_Optimization gets (data not needed bc only pea needs that which is the function!), #the function on which HP should be Optimized and search space for the HPs @@ -460,11 +461,16 @@ def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): #alpha = lambda x: mu(x) #negates alpha bc maximum has to be found alpha_neg = lambda x: -alpha(x) + x2 = np.linspace(0, 20, 21) + y2 = [alpha_neg(np.array([x_i, 0, 1, 0])) for x_i in x2] + plt.plot(x2, y2) bounds_and_x0 = self.get_bounds_and_x0() #bounds search space to vicinity of useful values bounds_an = bounds_and_x0[0] #problem: Nelder-Mead (same as fmin) cannot handle bounds -> use L-BFGS-B for now x0 = bounds_and_x0[1] new_x=minimize(alpha_neg, x0, method='L-BFGS-B', bounds=bounds_an).x + plt.title(str(new_x)) + plt.show() #print("new x: " + str(new_x) + " with function value: " + str(alpha(new_x))) print("acq_x returns: " + str(new_x) + " with a_neg(new_x) = " + str(alpha_neg(new_x))) print("mu(new_x) = " + str(mu(new_x)) + " , root(beta) = " + str(math.sqrt(beta)) + " , sigma(new_x) = " + str(sigma(new_x))) From 81ddcb620f8625fab2abf17dc9f0829f54ff2c6a Mon Sep 17 00:00:00 2001 From: Sonja Date: Sat, 4 Sep 2021 09:10:40 +0200 Subject: [PATCH 57/70] does not optimize lambda directly, but exponent of lambda = 10^-exp. Also hide plt for now --- sparseSpACE/HP_Optimization.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index e0b7a06..fa7432a 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -463,14 +463,14 @@ def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): alpha_neg = lambda x: -alpha(x) x2 = np.linspace(0, 20, 21) y2 = [alpha_neg(np.array([x_i, 0, 1, 0])) for x_i in x2] - plt.plot(x2, y2) + #plt.plot(x2, y2) bounds_and_x0 = self.get_bounds_and_x0() #bounds search space to vicinity of useful values bounds_an = bounds_and_x0[0] #problem: Nelder-Mead (same as fmin) cannot handle bounds -> use L-BFGS-B for now x0 = bounds_and_x0[1] new_x=minimize(alpha_neg, x0, method='L-BFGS-B', bounds=bounds_an).x - plt.title(str(new_x)) - plt.show() + #plt.title(str(new_x)) + #plt.show() #print("new x: " + str(new_x) + " with function value: " + str(alpha(new_x))) print("acq_x returns: " + str(new_x) + " with a_neg(new_x) = " + str(alpha_neg(new_x))) print("mu(new_x) = " + str(mu(new_x)) + " , root(beta) = " + str(math.sqrt(beta)) + " , sigma(new_x) = " + str(sigma(new_x))) @@ -537,8 +537,8 @@ def __init__(self, data_name: str = "moons", data = None, samples: int = 500, di self.data = data self.max_lv = max_lv self.max_evals = max_evals - self.classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1], ["list", 0, 1]] - self.class_dim_wise_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1], ["list", 0, 1, 2], ["interval", 0, 1], ["list", 0, 1], ["list", 0, 1], ["interval_int", 2, self.max_evals]] + self.classification_space = [["interval", 0, 20], ["list", 0, 1], ["list", 1], ["list", 0, 1]] + self.class_dim_wise_space = [["interval", 0, 20], ["list", 0, 1], ["list", 1], ["list", 0, 1, 2], ["interval", 0, 1], ["list", 0, 1], ["list", 0, 1], ["interval_int", 2, self.max_evals]] def create_data(self, name: str = "moons"): dataset = deml.datasets.make_moons(n_samples=self.samples, noise=0.15, random_state=1) @@ -571,7 +571,7 @@ def pea_classification(self, params): cur_data = self.data.copy() #should implement a smoother way to put in the data set classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) - classification.perform_classification(masslumping=params[1], lambd=params[0], minimum_level=params[2], maximum_level=self.max_lv, one_vs_others=params[3], print_metrics=False) + classification.perform_classification(masslumping=params[1], lambd=10**(-params[0]), minimum_level=params[2], maximum_level=self.max_lv, one_vs_others=params[3], print_metrics=False) evaluation = classification.evaluate() print("Percentage of correct mappings",evaluation["Percentage correct"]) #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion @@ -593,7 +593,7 @@ def pea_classification_dimension_wise(self, params): elif(params[3] == 1): ovo = True classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) - classification.perform_classification_dimension_wise(masslumping=params[1], lambd=params[0], minimum_level=params[2], maximum_level=self.max_lv, one_vs_others=ovo, error_calculator = ec, margin = params[4], rebalancing = params[5], use_relative_surplus = params[6], max_evaluations = params[7], print_metrics=False) + classification.perform_classification_dimension_wise(masslumping=params[1], lambd=10**(-params[0]), minimum_level=params[2], maximum_level=self.max_lv, one_vs_others=ovo, error_calculator = ec, margin = params[4], rebalancing = params[5], use_relative_surplus = params[6], max_evaluations = params[7], print_metrics=False) evaluation = classification.evaluate() print("Percentage of correct mappings",evaluation["Percentage correct"]) #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion @@ -610,7 +610,7 @@ def simple_test(params): #HPO = HP_Optimization(OC.pea_classification, OC.classification_space, r = 1.5) HPO = HP_Optimization(OC.pea_classification, OC.classification_space) #HPO = HP_Optimization(simple_test, simple_space, f_max = 100, r = 21) -HPO.perform_BO(5) +#HPO.perform_BO(5) #y_r = HPO.perform_RO(10)[3] #sol_b = HPO.perform_BO(6) #y_b = sol_b[3] From df48ca53ca93ab7e9254857815ae64da776d3676 Mon Sep 17 00:00:00 2001 From: Sonja Date: Mon, 6 Sep 2021 11:48:57 +0200 Subject: [PATCH 58/70] fixed mistake in get_beta and made it possible to set the l constant manually --- sparseSpACE/HP_Optimization.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index fa7432a..aea6114 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -22,6 +22,7 @@ def __init__(self, function, hp_space, f_max = None): self.delta = 0.1 self.a = 1.0 self.b = 1.0 + self.l = 0.5 def check_hp_space(self): if(len(self.hp_space)<1): @@ -64,7 +65,6 @@ def perform_GO(self, interval_levels = 4, search_space = None): best_x = None x_set = [] y_set = [] - #todo: split Dataset?? #original classification: classification = deml.Classification(data, split_percentage=0.8, split_evenly=True, shuffle_data=True) #original settings: classification.perform_classification(masslumping=True, lambd=0.0, minimum_level=1, maximum_level=5, print_metrics=True) #perform grid search for lambd_levels many lambd between 0 and 1 and all values for the rest @@ -75,7 +75,6 @@ def perform_GO(self, interval_levels = 4, search_space = None): search_space = self.cart_prod_hp_space(interval_levels) x_set = search_space for x in search_space: - #use "perform evaluation at" #cur_time = classification._time_used #print ("current time needed = " + str(cur_time)) cur_evaluation = self.perform_evaluation_at(x) @@ -372,13 +371,15 @@ def get_beta(self, t: int): delta: float = self.delta a: float = self.a b: float = self.b - beta = 2*math.log(t**2*2*math.pi**2/3*delta)+2*d*math.log(t**2*d*b*r*math.sqrt(math.log(4*d*a/delta))) + beta = 2*math.log(t**2*2*math.pi**2/(3*delta))+2*d*math.log(t**2*d*b*r*math.sqrt(math.log(4*d*a/delta))) print("get_beta returns: " + str(beta)) return beta - #TODO: get l_k and use it + #TODO: get optimized l_k in each iteration def get_l_k(self, t: int): - return 0.5 + return self.l + def set_l_k(self, l: float): + self.l = l #gets new beta and l if old beta and l give values that are already in C def get_new_beta_and_l(self, cur_beta: float, cur_amt_x, cur_x, C_x, C_y, amt_tries: int=0): From f36177a8e68a1c07482a2c276a60539c8d9876ce Mon Sep 17 00:00:00 2001 From: Sonja Date: Mon, 6 Sep 2021 14:05:57 +0200 Subject: [PATCH 59/70] changed .. something --- sparseSpACE/HP_Optimization.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index aea6114..4529a9f 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -408,9 +408,8 @@ def get_new_beta_and_l(self, cur_beta: float, cur_amt_x, cur_x, C_x, C_y, amt_tr #result is in the form [new beta, new l] return result - def get_bounds_and_x0(self): + def get_bounds_and_x0(self, offset = 0): bounds = [] - offset = 0 x0 = np.zeros(len(self.hp_space)) for i in range(0, len(self.hp_space)): new = [0, 1] @@ -462,8 +461,8 @@ def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): #alpha = lambda x: mu(x) #negates alpha bc maximum has to be found alpha_neg = lambda x: -alpha(x) - x2 = np.linspace(0, 20, 21) - y2 = [alpha_neg(np.array([x_i, 0, 1, 0])) for x_i in x2] + #x2 = np.linspace(0, 20, 21) + #y2 = [alpha_neg(np.array([x_i, 0, 1, 0])) for x_i in x2] #plt.plot(x2, y2) bounds_and_x0 = self.get_bounds_and_x0() #bounds search space to vicinity of useful values bounds_an = bounds_and_x0[0] From b2762d1f5580094d3931b5c0f0f58c669053a567 Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 7 Sep 2021 13:05:13 +0200 Subject: [PATCH 60/70] fixed random states of circles, moons, classification, blobs and gaussian quantiles data set. Also BO now also returns when random x was used --- sparseSpACE/HP_Optimization.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 4529a9f..93f3a02 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -271,7 +271,7 @@ def perform_BO(self, amt_it: int, r: float = 3.0, delta: float = 0.1, a: float = print("Random x has been used in iterations " + str(used_random_x)) print("C_y: " + str(C_y)) print("C_x: " + str(C_x)) - return x_ret, y_ret, C_x, C_y + return x_ret, y_ret, C_x, C_y, used_random_x #use squared exp. kernel as covariance function k, with parameters sigma_k and l_k sigma_k = 1 @@ -543,13 +543,13 @@ def __init__(self, data_name: str = "moons", data = None, samples: int = 500, di def create_data(self, name: str = "moons"): dataset = deml.datasets.make_moons(n_samples=self.samples, noise=0.15, random_state=1) if(name == "blobs"): - dataset = deml.datasets.make_blobs(n_samples=self.samples, n_features=self.dimension, centers=self.labels) + dataset = deml.datasets.make_blobs(n_samples=self.samples, n_features=self.dimension, centers=self.labels, random_state=1) elif(name == "circles"): - dataset = deml.datasets.make_circles(n_samples=self.samples, noise=0.05) + dataset = deml.datasets.make_circles(n_samples=self.samples, noise=0.05, random_state=1) elif(name == "classification"): - dataset = deml.datasets.make_classification(n_samples=self.samples, n_features=self.dimension, n_redundant=0, n_clusters_per_class=1, n_informative=2, n_classes=(self.labels if self.labels < 4 else 4)) + dataset = deml.datasets.make_classification(n_samples=self.samples, n_features=self.dimension, n_redundant=0, n_clusters_per_class=1, n_informative=2, n_classes=(self.labels if self.labels < 4 else 4), random_state=1) elif(name == "gaussian_quantiles"): - dataset = deml.datasets.make_gaussian_quantiles(n_samples=self.samples, n_features=self.dimension, n_classes=self.labels) + dataset = deml.datasets.make_gaussian_quantiles(n_samples=self.samples, n_features=self.dimension, n_classes=self.labels, random_state=1) elif(name == "digits"): dataset = deml.datasets.load_digits(return_X_y=True) # hint: try only with max_level <= 3 elif(name == "iris"): @@ -606,7 +606,7 @@ def simple_test(params): dataset_iris = deml.datasets.load_iris(return_X_y=True) iris_data = deml.DataSet(dataset_iris, name="data_iris") -OC = Optimize_Classification() +OC = Optimize_Classification(data_name = "moons") #HPO = HP_Optimization(OC.pea_classification, OC.classification_space, r = 1.5) HPO = HP_Optimization(OC.pea_classification, OC.classification_space) #HPO = HP_Optimization(simple_test, simple_space, f_max = 100, r = 21) From 2511f7da379d4c81f36213872e964b06f213c5cb Mon Sep 17 00:00:00 2001 From: Sonja Date: Tue, 7 Sep 2021 14:20:06 +0200 Subject: [PATCH 61/70] GO is now possible with int_lv = 0 --- sparseSpACE/HP_Optimization.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 93f3a02..36e4aca 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -96,7 +96,7 @@ def perform_GO(self, interval_levels = 4, search_space = None): def cart_prod_hp_space(self, interval_levels): res = [] - if (interval_levels == 0 or interval_levels == None): + if (interval_levels == None): interval_levels = 1 for i in range (0, len(self.hp_space)): new = [] @@ -111,7 +111,9 @@ def cart_prod_hp_space(self, interval_levels): new.append(self.hp_space[i][1]) elif(self.hp_space[i][0] == "interval"): h = self.hp_space[i][2]-self.hp_space[i][1] - dh = h /interval_levels + dh = h + if(interval_levels > 0): + dh = h / interval_levels #currently adds 1 more than interval_levels so that both borders are included - is there a better way to do this? if(h == 0): new.append(self.hp_space[i][1]) @@ -120,7 +122,9 @@ def cart_prod_hp_space(self, interval_levels): new.append(self.hp_space[i][1]+j*dh) elif(self.hp_space[i][0] == "interval_int"): h = int(self.hp_space[i][2]-self.hp_space[i][1]) - dh = h /interval_levels + dh = h + if (interval_levels>0): + dh = h /interval_levels #currently adds 1 more than interval_levels so that both borders are included - is there a better way to do this? if(h == 0): new.append(int(self.hp_space[i][1])) From 186a5e38cdbc67e170b6c212a96cc112e6ce3ae8 Mon Sep 17 00:00:00 2001 From: Sonja Date: Wed, 8 Sep 2021 16:21:43 +0200 Subject: [PATCH 62/70] interval_int now also only gives one value when interval_levels is 0 --- sparseSpACE/HP_Optimization.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 36e4aca..2873efd 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -132,8 +132,11 @@ def cart_prod_hp_space(self, interval_levels): if(dh<1): dh = 1 #may yield different values than expected due to rounding - for j in range (self.hp_space[i][1], self.hp_space[i][2]+1, int(dh)): - new.append(int(j)) + if(interval_levels == 0): + new.append(int(self.hp_space[i][1])) + else: + for j in range (self.hp_space[i][1], self.hp_space[i][2]+1, int(dh)): + new.append(int(j)) else: print("please enter valid types for hp_space! Using the first value given") new.append(self.hp_space[i][1]) From cc64a03fe8f72fb192b876585048a7ca18691e5e Mon Sep 17 00:00:00 2001 From: Sonja Date: Thu, 16 Sep 2021 11:55:55 +0200 Subject: [PATCH 63/70] all unnecessary prints should be removed --- sparseSpACE/HP_Optimization.py | 73 +++++++--------------------------- 1 file changed, 14 insertions(+), 59 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index 2873efd..d098572 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -79,18 +79,14 @@ def perform_GO(self, interval_levels = 4, search_space = None): #print ("current time needed = " + str(cur_time)) cur_evaluation = self.perform_evaluation_at(x) y_set.append(cur_evaluation) - print("Current Evaluation: ",cur_evaluation) #if best_evaluation == None or cur_evaluation > best_evaluation or (cur_evaluation == best_evaluation and (best_time == None or best_time>cur_time)): if(best_evaluation == None or cur_evaluation>best_evaluation): best_evaluation = cur_evaluation best_x = x - print("Best evaluation is now " + str(best_evaluation) + " at " + str(best_x)) - else: print("Best evaluation is still " + str(best_evaluation) + " at " + str(best_x)) - print ("END OF CURRENT EVALUATION \n") if(best_evaluation==self.f_max): - print("We've reached the specified maximum of f. Stopping GO.") + print("The specified maximum of f has been reached. Stopping GO.") break - print("In the end, best evaluation is " + str(best_evaluation) + " at " + str(best_x)) + print("The best evaluation found with grid search is " + str(best_evaluation) + " at " + str(best_x)) return best_x, best_evaluation, x_set, y_set #+ " and time = " + str(best_time)) @@ -101,13 +97,13 @@ def cart_prod_hp_space(self, interval_levels): for i in range (0, len(self.hp_space)): new = [] if(len(self.hp_space[i])<2): - print("please at least 1 value for hp_space list") + print("please enter at least 1 value for hp_space list") new.append(1) elif(self.hp_space[i][0] == "list"): new = self.hp_space[i].copy() new.remove("list") elif(len(self.hp_space[i])<3): - print("please enter at least 2 values for hp_space interval. Using the value given") + print("please enter at least 1 value for hp_space interval. Using the value given") new.append(self.hp_space[i][1]) elif(self.hp_space[i][0] == "interval"): h = self.hp_space[i][2]-self.hp_space[i][1] @@ -151,7 +147,6 @@ def perform_RO(self, amt_it: int): x_set = [] y_set = [] for i in range (0, amt_it, 1): - print("Random step " + str(i)) x = self.create_random_x() new_eval = self.perform_evaluation_at(x) x_set.append(x) @@ -160,7 +155,7 @@ def perform_RO(self, amt_it: int): best_x = x best_evaluation = new_eval if(best_evaluation == self.f_max): - print("We've reached the specified maximum of f. Stopping RO at iteration " + str(i)) + print("The specified maximum of f has been reached. Stopping RO at iteration " + str(i)) break print("Best evaluation in " + str(amt_it) + " random steps: " + str(best_evaluation) + " at " + str(best_x)) return best_x, best_evaluation, x_set, y_set @@ -173,9 +168,7 @@ def perform_evaluation_at(self, params): #print("Percentage of correct mappings",evaluation["Percentage correct"]) ##wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion #return evaluation["Percentage correct"] - print("performing evaluation at " + str(params)) res = self.function(params) - print("returning " + str(res)) return res #classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] @@ -192,83 +185,62 @@ def perform_BO(self, amt_it: int, r: float = 3.0, delta: float = 0.1, a: float = cur_amt_x: int = amt_HP+1 x_ret = None y_ret = None - print("I should do Bayesian Optimization for " + str(amt_it) + " iterations!") + print("Starting Bayesian Optimization with " + str(amt_it) + " iterations!") C = self.create_evidence_set(amt_it, amt_HP) C_x=C[0] - print("C_x: " + str(C_x)) C_y=C[1] - print("C_y: " + str(C_y)) #print("Evidence Set: \n" + str(C)) #For easier checking in what iterations random x has been used used_random_x = [] for i in range (0, amt_it, 1): - print("iteration: " + str(i)) beta = self.get_beta(i+1) l = self.get_l_k(i) - print("beta: " + str(beta) + ", cur_amt_x: " + str(cur_amt_x)) if(np.linalg.det(self.get_cov_matrix(C_x, cur_amt_x))==0): print("With the nex x value added the Covariance Matrix is singular.\nBayesian Optimization can not be continued and will be ended here, at the start of iteration " + str(i)) break if((self.get_cov_matrix(C_x, cur_amt_x) == np.identity(len(C_x))*(cur_amt_x+10)).all()): print("cov_matr has been modified to be Id*scalar! This means it was singular -> ending at iteration " + str(i)) break - else: - print("Apparently the cov_matr is not singular at the beginning of iteration " + str(i)) #value that will be evaluated and added to the evidence set - print("Getting new x:") new_x=self.acq_x(beta, l, C_x, C_y, cur_amt_x) - print("acq_x gives new x = " + str(new_x)) new_x_rd = self.round_x(new_x) old_x_rd = None old_beta_and_l = [beta, l] - print("new x rd: " + str(new_x_rd)) #erstelle neues beta und l und berechne neuen wert damit amt_tries = 0 while(self.check_if_in_array(new_x_rd, C_x)): - print("!!!!!!!!!!!Need new beta and l") + print("x is already included in C") old_x_rd = new_x_rd.copy() - print("old x rd was: " + str(old_x_rd)) new_beta_and_l = self.get_new_beta_and_l(beta, cur_amt_x, new_x, C_x, C_y, amt_tries = amt_tries) amt_tries += 1 new_x = self.acq_x(new_beta_and_l[0], new_beta_and_l[1], C_x, C_y, cur_amt_x) new_x_rd = self.round_x(new_x) - print("new x rd is: " + str(new_x_rd)) if(old_x_rd==new_x_rd and amt_tries >= 3): - print("It seems a beta and l that give a new x cannot be found. Trying beta = 0 and l = 0.5") new_x = self.acq_x(0, 0.5, C_x, C_y, cur_amt_x) new_x_rd = self.round_x(new_x) if(self.check_if_in_array(new_x_rd, C_x)): - print("Beta = 0 didn't help. Using random new x and then getting out.") - print("If a random x that's not in C_x can not be found in 10 tries it is assumed that all possible x have been used already and BO is stopped") used_random_x.append(i) while(self.check_if_in_array(new_x_rd, C_x) and amt_tries < 13): new_x_rd = self.create_random_x() amt_tries += 1 break - else: - print("Beta = 0 DID help!") - print("out of getting beta and l loop") if(old_x_rd==new_x_rd or self.check_if_in_array(new_x_rd, C_x)): - print("There was an infinite loop when getting new x and / or no new x has been found. Ending BO at iteration " + str(i)) + print("No new x has been found. Ending BO at iteration " + str(i)) break - print("adding " + str(new_x_rd)) C_x[cur_amt_x] = new_x_rd C_y[cur_amt_x] = self.perform_evaluation_at(new_x_rd) cur_amt_x += 1 if(C_y[cur_amt_x-1]==self.f_max): - print("We've reached the specified maximum of f. Stopping BO at iteration " + str(i)) + print("The specified maximum of f has been reached. Stopping BO at iteration " + str(i)) break #ends everything if cov_matr is singular. Should this be dependant on l? Cov_matr being singular should not depend on l I think - print("Checking if cov_matr is singular") if(np.linalg.det(self.get_cov_matrix(C_x, cur_amt_x)) == 0): print("With the nex x value added the Covariance Matrix is singular.\nBayesian Optimization can not be continued and will be ended here, after " + str(i) + " iterations") break if((self.get_cov_matrix(C_x, cur_amt_x) == np.identity(len(C_x))*(cur_amt_x+10)).all()): print("cov_matr has been modified to be Id*scalar! This means it was singular -> ending at iteration " + str(i)) break - else: - print("Apparently the cov_matr is not singular after iteration " + str(i)) for i in range(0, cur_amt_x, 1): if(y_ret == None or y_ret=1) #print(K_matr) mu = lambda x: self.get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(C_y)) @@ -479,15 +441,10 @@ def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): #plt.title(str(new_x)) #plt.show() #print("new x: " + str(new_x) + " with function value: " + str(alpha(new_x))) - print("acq_x returns: " + str(new_x) + " with a_neg(new_x) = " + str(alpha_neg(new_x))) - print("mu(new_x) = " + str(mu(new_x)) + " , root(beta) = " + str(math.sqrt(beta)) + " , sigma(new_x) = " + str(sigma(new_x))) - print("sigma_sqrd(new_x) = " + str(sigma_sqrd(new_x))) - print("k_vec(x) = " + str(self.get_cov_vector(C_x, new_x, cur_amt_x))) return new_x #needs search space def round_x(self, x): - print("rounding x") if len(x) < len(self.hp_space): print("Input too short! Returning default values rd(0) for missing values") for k in range (len(x), len(self.hp_space)): @@ -566,13 +523,13 @@ def create_data(self, name: str = "moons"): elif(name == "wine"): dataset = deml.datasets.load_wine(return_X_y=True) elif(name != "moons"): - print("This is not a valid name for a dataset, using moons") + print("This name for the data set is unknown, using moons.\nPlease check that the name is not capitalized, words are separated by _, and that it is written correctly") data = deml.DataSet(dataset, name="data_"+name) return data def pea_classification(self, params): if(len(params)<4): - print("too little params for pea_classification. Returning 0.0") + print("too little parameters for pea_classification. Returning 0.0") return 0.0 params = [params[0], int(params[1]), int(params[2]), int(params[3])] cur_data = self.data.copy() @@ -580,13 +537,12 @@ def pea_classification(self, params): classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) classification.perform_classification(masslumping=params[1], lambd=10**(-params[0]), minimum_level=params[2], maximum_level=self.max_lv, one_vs_others=params[3], print_metrics=False) evaluation = classification.evaluate() - print("Percentage of correct mappings",evaluation["Percentage correct"]) #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion return evaluation["Percentage correct"] def pea_classification_dimension_wise(self, params): if(len(params)<4): - print("too little params for pea_classification. Returning 0.0") + print("too little parameters for pea_classification. Returning 0.0") return 0.0 #lambd, massl, min_lv, one_vs_others / error calc, margin(float 0-1), rebalancing (bool), use_relative_surplus (bool), max_evaluations (int) params = [float(params[0]), int(params[1]), int(params[2]), int(params[3]), float(params[4]), int(params[5]), int(params[6]), int(params[7])] @@ -602,7 +558,6 @@ def pea_classification_dimension_wise(self, params): classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) classification.perform_classification_dimension_wise(masslumping=params[1], lambd=10**(-params[0]), minimum_level=params[2], maximum_level=self.max_lv, one_vs_others=ovo, error_calculator = ec, margin = params[4], rebalancing = params[5], use_relative_surplus = params[6], max_evaluations = params[7], print_metrics=False) evaluation = classification.evaluate() - print("Percentage of correct mappings",evaluation["Percentage correct"]) #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion return evaluation["Percentage correct"] From 0d33edc3f94c062e09560028e119f8db8b04fbc7 Mon Sep 17 00:00:00 2001 From: Sonja Date: Mon, 1 Nov 2021 11:26:23 +0100 Subject: [PATCH 64/70] added test_HP_Optimization file (?) --- test/test_HP_Optimization.py | 46 ++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 test/test_HP_Optimization.py diff --git a/test/test_HP_Optimization.py b/test/test_HP_Optimization.py new file mode 100644 index 0000000..4633d35 --- /dev/null +++ b/test/test_HP_Optimization.py @@ -0,0 +1,46 @@ +import unittest +import sparseSpACE +import sparseSpACE.HP_Optimization as hpo +import numpy as np +import math + +class TestHP_Optimization(unittest.TestCase): + def test_HPO(self): + OC = hpo.Optimize_Classification(data_name = "moons") + HPO = hpo.HP_Optimization(OC.pea_classification, OC.classification_space) + #test beta + beta_1 = HPO.get_beta(1) + self.assertAlmostEqual(beta_1, 34.74985580606742) + #test cov + v1 = np.array([1, 2]) + v2 = np.array([3, 4]) + res = HPO.cov(v1, v2) + self.assertAlmostEqual(res, math.exp(-16)) + #test cov_matrix + C_x=np.array([[1, 2], [3, 4]]) + K = HPO.get_cov_matrix(C_x, 2) + control_K = np.array([[1, math.exp(-16)], [math.exp(-16), 1]]) + np.testing.assert_almost_equal(K, control_K) + #test cov_vector + C_x=np.array([[1, 2], [5, 6]]) + x = np.array([3, 4]) + v = HPO.get_cov_vector(C_x, x, 2) + control_v = np.array([math.exp(-16), math.exp(-64)]) + np.testing.assert_almost_equal(v, control_v) + simple_function = lambda x: x[0]+x[1] + simple_space = [["list", 0.42, 1, 2.5, 3.3], ["list", 2, 4]] + HPO_simple = hpo.HP_Optimization(simple_function, simple_space) + #test GO + res_GO = HPO_simple.perform_GO() + np.testing.assert_almost_equal(res_GO[0], [3.3, 4]) + self.assertAlmostEqual(res_GO[1], 7.3) + ##test RO - should I even test RO?? It's usually not that close to the optimum + message = "The optimum found by perform_RO is not close enough to the actual optimum" + res_RO = HPO_simple.perform_RO(15) + np.testing.assert_almost_equal(res_RO[0], [3.3, 4], 1, message) + self.assertAlmostEqual(res_RO[1], 7.3, 2) + def simple_function(self, x): + return x[0]+x[1] + + +unittest.main() From f9e6b9578858e0e17eb67835936cccc2b5a8dfe8 Mon Sep 17 00:00:00 2001 From: Sonja Date: Mon, 1 Nov 2021 11:33:13 +0100 Subject: [PATCH 65/70] added test for getting random x (whether criteria are held) --- test/test_HP_Optimization.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/test/test_HP_Optimization.py b/test/test_HP_Optimization.py index 4633d35..1a5e0a4 100644 --- a/test/test_HP_Optimization.py +++ b/test/test_HP_Optimization.py @@ -35,10 +35,16 @@ def test_HPO(self): np.testing.assert_almost_equal(res_GO[0], [3.3, 4]) self.assertAlmostEqual(res_GO[1], 7.3) ##test RO - should I even test RO?? It's usually not that close to the optimum - message = "The optimum found by perform_RO is not close enough to the actual optimum" - res_RO = HPO_simple.perform_RO(15) - np.testing.assert_almost_equal(res_RO[0], [3.3, 4], 1, message) - self.assertAlmostEqual(res_RO[1], 7.3, 2) + #message = "The optimum found by perform_RO is not close enough to the actual optimum" + #res_RO = HPO_simple.perform_RO(15) + #np.testing.assert_almost_equal(res_RO[0], [3.3, 4], 1, message) + #self.assertAlmostEqual(res_RO[1], 7.3, 2) + + #test get random x + x = HPO_simple.create_random_x() + x_rd = HPO_simple.round_x(x) + self.assertEqual(x, x_rd) + def simple_function(self, x): return x[0]+x[1] From a68a373ad6787baa8007217b74e1b2c8cacf606b Mon Sep 17 00:00:00 2001 From: Sonja Date: Fri, 19 Nov 2021 10:17:22 +0100 Subject: [PATCH 66/70] Added test for get_bounds_and_x0 --- test/test_HP_Optimization.py | 64 ++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/test/test_HP_Optimization.py b/test/test_HP_Optimization.py index 1a5e0a4..844e9ad 100644 --- a/test/test_HP_Optimization.py +++ b/test/test_HP_Optimization.py @@ -8,7 +8,15 @@ class TestHP_Optimization(unittest.TestCase): def test_HPO(self): OC = hpo.Optimize_Classification(data_name = "moons") HPO = hpo.HP_Optimization(OC.pea_classification, OC.classification_space) - #test beta + #test check_if_in_array (only works with np.array) + array = np.array([[0, 1, 1, 1], [7, 3, 6, 5], [4.2, 3.3, 5.2, 9], [7, 7, 7, 7]]) + b = HPO.check_if_in_array([7, 7, 7, 7], array) + print("b = " + str(b)) + self.assertTrue(b) + self.assertTrue(HPO.check_if_in_array([4.2, 3.3, 5.2, 9], array)) + self.assertFalse(HPO.check_if_in_array([1, 0, 0, 0], array)) + + #test get_beta beta_1 = HPO.get_beta(1) self.assertAlmostEqual(beta_1, 34.74985580606742) #test cov @@ -27,10 +35,31 @@ def test_HPO(self): v = HPO.get_cov_vector(C_x, x, 2) control_v = np.array([math.exp(-16), math.exp(-64)]) np.testing.assert_almost_equal(v, control_v) + #test check_hp_space simple_function = lambda x: x[0]+x[1] - simple_space = [["list", 0.42, 1, 2.5, 3.3], ["list", 2, 4]] + simple_space = [] + cmp_space = [["list", 1]] + HPO_simple = hpo.HP_Optimization(simple_function, simple_space) + self.assertEqual(HPO_simple.hp_space, cmp_space) + simple_space = [["interval", 5, 3.3], [7], ["interval_int", 4.2, 1.1], ["interval", 7]] + cmp_space = [["interval", 3.3, 5], ["list", 1], ["interval_int", 1, 4], ["interval", 7, 7]] HPO_simple = hpo.HP_Optimization(simple_function, simple_space) + self.assertEqual(HPO_simple.hp_space, cmp_space) + simple_space = [["list", 0.42, 3.3], ["interval", 2, 4], ["interval_int", 2, 4]] + HPO_simple = hpo.HP_Optimization(simple_function, simple_space) + self.assertEqual(HPO_simple.hp_space, [["list", 0.42, 3.3], ["interval", 2, 4], ["interval_int", 2, 4]]) + #test cart_prod_hp_space + cart_prod = HPO_simple.cart_prod_hp_space(1) + cmp_cart_prod = [(0.42, 2.0, 2), (0.42, 2.0, 4), (0.42, 4.0, 2), (0.42, 4.0, 4), (3.3, 2.0, 2), (3.3, 2.0, 4), (3.3, 4.0, 2), (3.3, 4.0, 4)] + self.assertEqual(cart_prod, cmp_cart_prod) + #test perform_eval_at + res = HPO_simple.perform_evaluation_at([0.42, 4.0, 2]) + self.assertEqual(res, 4.42) + res2 = HPO_simple.perform_evaluation_at([42, 1337]) + self.assertEqual(res2, 1379) #test GO + simple_space = [["list", 0.42, 1, 2.5, 3.3], ["list", 2, 4]] + HPO_simple = hpo.HP_Optimization(simple_function, simple_space) res_GO = HPO_simple.perform_GO() np.testing.assert_almost_equal(res_GO[0], [3.3, 4]) self.assertAlmostEqual(res_GO[1], 7.3) @@ -39,11 +68,40 @@ def test_HPO(self): #res_RO = HPO_simple.perform_RO(15) #np.testing.assert_almost_equal(res_RO[0], [3.3, 4], 1, message) #self.assertAlmostEqual(res_RO[1], 7.3, 2) - + + #test round x + simple_space = [["list", 0.42, 1, 2.5, 3.3], ["interval", 2, 4], ["interval_int", 1, 5]] + HPO_simple = hpo.HP_Optimization(simple_function, simple_space) + x = [0.4, 3.33, 2.7] + x_rd = HPO_simple.round_x(x) + y = [2.5, 5, 5] + y_rd = HPO_simple.round_x(y) + self.assertEqual(x_rd, [0.42, 3.33, 3]) + self.assertEqual(y_rd, [2.5, 4, 5]) #test get random x x = HPO_simple.create_random_x() x_rd = HPO_simple.round_x(x) self.assertEqual(x, x_rd) + #test create_evidence_set + C = HPO_simple.create_evidence_set(5, 3) + self.assertEqual(len(C[0]), 9) + self.assertEqual(len(C[0][0]), 3) + self.assertEqual(HPO_simple.perform_evaluation_at(C[0][0]), C[1][0]) + cmp = [0, 0, 0] + np.testing.assert_equal(C[0][7], cmp) + #test get_bounds_and_x0 + #bounds are supposed to be smallest and biggest possible values, x0 consists of smallest values + bounds_and_x0 = HPO_simple.get_bounds_and_x0(0) + self.assertEqual(bounds_and_x0[0][0][0], 0.42) + self.assertEqual(bounds_and_x0[0][0][1], 3.3) + self.assertEqual(bounds_and_x0[0][1][0], 2) + self.assertEqual(bounds_and_x0[0][1][1], 4) + self.assertEqual(bounds_and_x0[0][2][0], 1) + self.assertEqual(bounds_and_x0[0][2][1], 5) + + self.assertEqual(bounds_and_x0[1][0], 0.42) + self.assertEqual(bounds_and_x0[1][1], 2) + self.assertEqual(bounds_and_x0[1][2], 1) def simple_function(self, x): return x[0]+x[1] From 3fc324f311ecdb416f2d16fbdcf7a4d322ba9ebf Mon Sep 17 00:00:00 2001 From: Sonja Date: Fri, 19 Nov 2021 10:33:27 +0100 Subject: [PATCH 67/70] deleted test text file --- Test_Sonja.txt | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 Test_Sonja.txt diff --git a/Test_Sonja.txt b/Test_Sonja.txt deleted file mode 100644 index 09c235e..0000000 --- a/Test_Sonja.txt +++ /dev/null @@ -1,2 +0,0 @@ -Hallo Sonja! -OMG das bin ja ich! From 468ca989b562d4e616f992949dfdeb8046fe0bc8 Mon Sep 17 00:00:00 2001 From: Sonja Date: Fri, 19 Nov 2021 13:49:00 +0100 Subject: [PATCH 68/70] added "if.." to test, more comments to Optimize_Classification, removed unused test stuff at the end of HP_optimization --- sparseSpACE/HP_Optimization.py.save | 107 ---------------------------- test/test_HP_Optimization.py | 11 +-- 2 files changed, 2 insertions(+), 116 deletions(-) delete mode 100644 sparseSpACE/HP_Optimization.py.save diff --git a/sparseSpACE/HP_Optimization.py.save b/sparseSpACE/HP_Optimization.py.save deleted file mode 100644 index 17f96c5..0000000 --- a/sparseSpACE/HP_Optimization.py.save +++ /dev/null @@ -1,107 +0,0 @@ -import sparseSpACE.DEMachineLearning as deml -import numpy as np -import scipy as sp -import math - -#performs Grid Optimization -def perform_GO_classification(data): - #sklearn_dataset = deml.datasets.make_moons(n_samples=samples, noise=0.15) - #data = deml.DataSet(sklearn_dataset, name='Input_Set') - #for storing the current best evaluation and time - best_evaluation = None - best_time = None - #storing the parameters of the best evaluation - best_lambd = None - best_masslump = None - best_min_lv = None - best_one_vs_others = None - #For Testing best evaluation: best_evaluation = 0.7 - #print("original best evaluation: " + str(best_evaluation)) - #todo: split Dataset - #original classification: classification = deml.Classification(data, split_percentage=0.8, split_evenly=True, shuffle_data=True) - #original settings: classification.perform_classification(masslumping=True, lambd=0.0, minimum_level=1, maximum_level=5, print_metrics=True) - #perform grid search for a few values of masslumping and lambd - lambd_levels = 10 - #Note: vllt besser logarithmisch, sehr kleine Werte oft besser (1/10, 1/100, 1/1000) - for cur_lambd in range (0, lambd_levels+1, 5): - #Note: lamd vor Allem sinnvoll wenn masslumping aus ist, sollte kaum unterschied machen wenn - for cur_massl in [True, False]: - for cur_min_lv in range (1, 4, 1): - for cur_one_vs_others in [True, False]: - classification = deml.Classification(data, split_percentage=0.8, split_evenly=True, shuffle_data=False) - print("current masslumping = " + str(cur_massl) + "; current lambd = " + str(cur_lambd/lambd_levels) + "; current min lv = " + str(cur_min_lv) + "; current one vs others = " + str(cur_one_vs_others)) - classification.perform_classification(masslumping=cur_massl, lambd=cur_lambd/lambd_levels, minimum_level=cur_min_lv, maximum_level=5, one_vs_others=cur_one_vs_others, print_metrics=False) - cur_time = classification._time_used - print ("current time needed = " + str(cur_time)) - evaluation = classification.evaluate() - print("Percentage of correct mappings",evaluation["Percentage correct"]) - #currently only looking at Percentage correct and time needed for evaluating - cur_evaluation = evaluation["Percentage correct"] - if best_evaluation == None or cur_evaluation > best_evaluation or (cur_evaluation == best_evaluation and (best_time == None or best_time>cur_time)): - best_evaluation = cur_evaluation - best_lambd = cur_lambd - best_time = cur_time - best_masslump = cur_massl - best_min_lv = cur_min_lv - best_one_vs_others = cur_one_vs_others - print("Best evaluation is now " + str(best_evaluation) + " with masslump = " + str(best_masslump) + ", lambd = " + str(best_lambd) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others) + " and time = " + str(best_time)) - else: print("Best evaluation is still " + str(best_evaluation) + " with masslump = " + str(best_masslump) + ", lambd = " + str(best_lambd) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others) + " and time = " + str(best_time)) - print ("END OF CURRENT EVALUATION \n") - print("In the end, best evaluation is " + str(best_evaluation) + " with masslump = " + str (best_masslump) + ", lamb = " + str(best_lambd) + ", min_level = " + str(best_min_lv) + ", one_vs_others = " + str(best_one_vs_others) + " and time = " + str(best_time)) - -#returns evaluation for certain Parameters on a certain data set -def perform_evaluation_at(cur_data, cur_lambd: float, cur_massl: bool, cur_min_lv: int, cur_one_vs_others: bool): - classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) - classification.perform_classification(masslumping=cur_massl, lambd=cur_lambd, minimum_level=cur_min_lv, maximum_level=5, one_vs_others=cur_one_vs_others, print_metrics=False) - evaluation = classification.evaluate() - print("Percentage of correct mappings",evaluation["Percentage correct"]) - return evaluation["Percentage correct"] - -#TODO: will perform Bayesian Optimization; Inputs are the data, amount of iterations(amt_it) -def perform_BO_classification(data, amt_it: int, dim: int): - - print("I should do Bayesian Optimization for " + str(amt_it) + " iteration!") - #dummy value for beta - beta = 0.5 - #use squared exp. kernel as covariance function k, with parameters sigma_k and l_k - sigma_k = 1 - l_k = 0.5 - #if creation of covariance matrices is done in another function k(x) needs to be outside - k = lambda x, y: (sigma_k**2)*math.exp((-0.5/l_k**2)*np.linalg.norm(x-y)**2) - mu = lambda x: x**2+4 - sigma_sqrd = lambda x: k(x, x) - sigma = lambda x: math.sqrt(sigma_sqrd(x)) - alpha = lambda x: mu(x)+(math.sqrt(beta))*sigma(x) - for i in range (0, amt_it, 1): - beta = get_beta(i) - print("beta: " + str(beta) + " mu(x): " + str(mu(2)) + " sigma(x): " + str(sigma(2))) - print(alpha(2)) - -#TODO: will calculate and return current beta -def get_beta(cur_it: int): - beta = cur_it/2 - return beta - -#TODO: will return initial evidence set -def get_init_evidence(data, dim: int, amt_it: int): - #hard-coded x-values. Order:[lambd: float, massl:bool, min_lv: 0 Date: Fri, 19 Nov 2021 14:22:26 +0100 Subject: [PATCH 69/70] comments on pea_methods --- sparseSpACE/HP_Optimization.py | 57 ++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index d098572..cdbcb48 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -491,6 +491,16 @@ def round_x(self, x): return new_x_rd class Optimize_Classification: + """Constructor of the Optimize_Classification class. + + Takes in: + :param data_name: The name of a data set to create such a data set. + Options are: moons, blobs, circles, classification, gaussian_quantiles, + digits, iris, breast_cancer, wine. Can also take in a dataset directly. + :param data: Optional. If no data set is given here data is created by name, using "moons" as default + :param max_lv & max_evals: Parameters of classification and class_dim_wise that are not otimized + but can be specified here + """ def __init__(self, data_name: str = "moons", data = None, samples: int = 500, dimension: int = 2, labels: int = 6, max_lv: int = 3, max_evals: int = 256): self.samples = samples self.dimension = dimension @@ -527,19 +537,38 @@ def create_data(self, name: str = "moons"): data = deml.DataSet(dataset, name="data_"+name) return data + """Perform classification with certain HP-input and return evaluation + Parameters are: + :params[0]: lambd_exp: used to calculate lambda = 10**(-lambd_exp) + :params[1]: massl + :params[2]: min_lv + :params[3]: ove_vs_others + """ def pea_classification(self, params): if(len(params)<4): print("too little parameters for pea_classification. Returning 0.0") return 0.0 params = [params[0], int(params[1]), int(params[2]), int(params[3])] cur_data = self.data.copy() - #should implement a smoother way to put in the data set classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) classification.perform_classification(masslumping=params[1], lambd=10**(-params[0]), minimum_level=params[2], maximum_level=self.max_lv, one_vs_others=params[3], print_metrics=False) evaluation = classification.evaluate() - #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion return evaluation["Percentage correct"] + """Perform dimension wise classification with certain HP-input and return evaluation + Parameters are: + :params[0]: lambd_exp: used to calculate lambda = 10**(-lambd_exp) + :params[1]: massl + :params[2]: min_lv + :params[3]: ove_vs_others / error calculator. + case 0: ovo=false, ec=none; + case1: ovo=true, ec=none; + case2: ovo=true, ec=error_calculator + :params[4]: margin + :params[5]: rebalancing + :params[6]: use_relative_surplus + :params[7]: max_evaluations + """ def pea_classification_dimension_wise(self, params): if(len(params)<4): print("too little parameters for pea_classification. Returning 0.0") @@ -561,28 +590,4 @@ def pea_classification_dimension_wise(self, params): #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion return evaluation["Percentage correct"] -#anderes dataset ausprobieren?, vllt höhere dimensionen, bis zu dim=10. Note: moons immer 2d, aber z.B. -def simple_test(params): - return params[0] -simple_space = [["interval", 20.4, 0.]] - -dataset_iris = deml.datasets.load_iris(return_X_y=True) -iris_data = deml.DataSet(dataset_iris, name="data_iris") -OC = Optimize_Classification(data_name = "moons") -#HPO = HP_Optimization(OC.pea_classification, OC.classification_space, r = 1.5) -HPO = HP_Optimization(OC.pea_classification, OC.classification_space) -#HPO = HP_Optimization(simple_test, simple_space, f_max = 100, r = 21) -#HPO.perform_BO(5) -#y_r = HPO.perform_RO(10)[3] -#sol_b = HPO.perform_BO(6) -#y_b = sol_b[3] -#x_b = sol_b[2] -#print("y random: " + str(y_r)) -#print("y bayesian: " + str(y_b)) -#print("x bayesian: " + str(x_b)) -#print(OC.pea_classification_dimension_wise([0.0, 0, 1, 0, 0.0, 0, 0, 2])) - -#HPO = HP_Optimization(simple_test, simple_space) -#print(HPO.hp_space) -#HPO.perform_GO(search_space=[[1, 1, 2, 1], [5, 4, 2, 0], [3, 3, 2, 1]]) From f05c8209e3eab047882320e4514fda5fba597770 Mon Sep 17 00:00:00 2001 From: Sonja Date: Fri, 19 Nov 2021 17:01:17 +0100 Subject: [PATCH 70/70] Refined / added comments --- sparseSpACE/HP_Optimization.py | 161 +++++++++++++++------------------ 1 file changed, 72 insertions(+), 89 deletions(-) diff --git a/sparseSpACE/HP_Optimization.py b/sparseSpACE/HP_Optimization.py index cdbcb48..b4e8f0d 100644 --- a/sparseSpACE/HP_Optimization.py +++ b/sparseSpACE/HP_Optimization.py @@ -9,10 +9,17 @@ from scipy.optimize import minimize import matplotlib.pyplot as plt -#class HP_Optimization gets (data not needed bc only pea needs that which is the function!), -#the function on which HP should be Optimized and search space for the HPs -#space is in form [["interval", , ], ["list", , , ,...], ....] + class HP_Optimization: + """Constructor of HP_Optimization class + :param function: The function to be optimized / find the maximum of + :param hp_space: Defines the possible input values of the function. Input values are assumed to be numerical. + Array that contains another array for each parameter. Can be one of three types: + 1.["interval", min, max] + 2.["interval_int", min, max] + 3.["list", value_1, value_2, ..., value_n] + :param f_max: Optional. Maximal value of the function. Optimization is stopped if this value is reached. + """ def __init__(self, function, hp_space, f_max = None): self.function = function self.hp_space = hp_space @@ -24,6 +31,9 @@ def __init__(self, function, hp_space, f_max = None): self.b = 1.0 self.l = 0.5 + """checks whether hp_space fulfills certain criteria (e.g. enough values, minmax -> max=min, min=max) or adds default values when needed. + """ def check_hp_space(self): if(len(self.hp_space)<1): print("Please enter values for hp_space! Using default value ['list', 1], but will likely throw errors") @@ -54,32 +64,29 @@ def check_hp_space(self): print("Unknown descriptor for hp_space at index " + str(i) +". Using ['list', ]") self.hp_space[i] = ["list", self.hp_space[i][1]] - #performs Grid Optimization + """Perform Grid Optimization + :param interval_levels: Optional. Default value: 4. If no search space is given, search space is generated by taking a subset + of values for each parameter and taking the cartesian product of these subsets. + For "list"-type parameters are values are taken. From "interval"-type parameters roughly "interval_levels"+1 many + evenly split values are taken (e.g. from ["interval", 0, 10] take [0, 1, 2, ..., 10]). + If taking this many values not possible (e.g. 3 integers between 0 and 1), takes as many values as possible + :param search_space: Optional. Directly give all the parameter values that should be tried for grid search. + : + """ def perform_GO(self, interval_levels = 4, search_space = None): - #sklearn_dataset = deml.datasets.make_moons(n_samples=samples, noise=0.15) - #data = deml.DataSet(sklearn_dataset, name='Input_Set') - #for storing the current best evaluation and time + #for storing the current best evaluation best_evaluation = None - #best_time = None #storing the parameters of the best evaluation best_x = None + #storing all x and their respective y x_set = [] y_set = [] - #original classification: classification = deml.Classification(data, split_percentage=0.8, split_evenly=True, shuffle_data=True) - #original settings: classification.perform_classification(masslumping=True, lambd=0.0, minimum_level=1, maximum_level=5, print_metrics=True) - #perform grid search for lambd_levels many lambd between 0 and 1 and all values for the rest - #Note: lambd vllt besser logarithmisch, sehr kleine Werte oft besser (1/10, 1/100, 1/1000) - #für for-schleife karthesisches produkt aus HO um alle tupel zu bilden und über diese tupel zu iterieren für beliebig dimensional - #Note: lambd vor Allem sinnvoll wenn masslumping aus ist, sollte kaum unterschied machen wenn an if(search_space == None): search_space = self.cart_prod_hp_space(interval_levels) x_set = search_space for x in search_space: - #cur_time = classification._time_used - #print ("current time needed = " + str(cur_time)) cur_evaluation = self.perform_evaluation_at(x) y_set.append(cur_evaluation) - #if best_evaluation == None or cur_evaluation > best_evaluation or (cur_evaluation == best_evaluation and (best_time == None or best_time>cur_time)): if(best_evaluation == None or cur_evaluation>best_evaluation): best_evaluation = cur_evaluation best_x = x @@ -88,8 +95,8 @@ def perform_GO(self, interval_levels = 4, search_space = None): break print("The best evaluation found with grid search is " + str(best_evaluation) + " at " + str(best_x)) return best_x, best_evaluation, x_set, y_set - #+ " and time = " + str(best_time)) + #creates the search space as explained in explanation of param interval_levels of method perform_GO def cart_prod_hp_space(self, interval_levels): res = [] if (interval_levels == None): @@ -140,7 +147,9 @@ def cart_prod_hp_space(self, interval_levels): res_cart = list(itertools.product(*res)) return res_cart - #performs random optimization + """Performs random optimization + :param amt_it: the amount of iterations random search is performed for + """ def perform_RO(self, amt_it: int): best_evaluation = 0 best_x = None @@ -160,21 +169,19 @@ def perform_RO(self, amt_it: int): print("Best evaluation in " + str(amt_it) + " random steps: " + str(best_evaluation) + " at " + str(best_x)) return best_x, best_evaluation, x_set, y_set - #returns evaluation for certain Parameters on a certain data set + #returns evaluation of the given function for certain Parameters on a certain data set def perform_evaluation_at(self, params): - #classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) - #classification.perform_classification(masslumping=cur_massl, lambd=cur_lambd, minimum_level=cur_min_lv, maximum_level=5, one_vs_others=cur_one_vs_others, print_metrics=False) - #evaluation = classification.evaluate() - #print("Percentage of correct mappings",evaluation["Percentage correct"]) - ##wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion - #return evaluation["Percentage correct"] res = self.function(params) return res - #classification_space = [["interval", 0, 1], ["list", 0, 1], ["list", 1, 2, 3], ["list", 0, 1]] - #performs Bayesian Optimization - #Note: HP Input of classification is (lambd:float, massl:bool, min_lv:int <4, one_vs_others:bool) + """performs Bayesian Optimization + :param amt_it: The amount of iterations Bayesian Optimization is performed for AFTER creating the initial evidence set + :param r: Parameter for the computation of beta + :param delta: see r + :param a: see r + :param b: see r + """ def perform_BO(self, amt_it: int, r: float = 3.0, delta: float = 0.1, a: float = 1.0, b: float = 1.0): self.r = r self.delta = delta @@ -189,7 +196,6 @@ def perform_BO(self, amt_it: int, r: float = 3.0, delta: float = 0.1, a: float = C = self.create_evidence_set(amt_it, amt_HP) C_x=C[0] C_y=C[1] - #print("Evidence Set: \n" + str(C)) #For easier checking in what iterations random x has been used used_random_x = [] for i in range (0, amt_it, 1): @@ -202,7 +208,7 @@ def perform_BO(self, amt_it: int, r: float = 3.0, delta: float = 0.1, a: float = print("cov_matr has been modified to be Id*scalar! This means it was singular -> ending at iteration " + str(i)) break - #value that will be evaluated and added to the evidence set + #new x value that will be evaluated and added to the evidence set new_x=self.acq_x(beta, l, C_x, C_y, cur_amt_x) new_x_rd = self.round_x(new_x) old_x_rd = None @@ -234,7 +240,7 @@ def perform_BO(self, amt_it: int, r: float = 3.0, delta: float = 0.1, a: float = if(C_y[cur_amt_x-1]==self.f_max): print("The specified maximum of f has been reached. Stopping BO at iteration " + str(i)) break - #ends everything if cov_matr is singular. Should this be dependant on l? Cov_matr being singular should not depend on l I think + #ends everything if cov_matr is singular if(np.linalg.det(self.get_cov_matrix(C_x, cur_amt_x)) == 0): print("With the nex x value added the Covariance Matrix is singular.\nBayesian Optimization can not be continued and will be ended here, after " + str(i) + " iterations") break @@ -253,30 +259,25 @@ def perform_BO(self, amt_it: int, r: float = 3.0, delta: float = 0.1, a: float = #use squared exp. kernel as covariance function k, with parameters sigma_k and l_k sigma_k = 1 l_k = 0.5 - #since creation of covariance matrices is done in another function k(x) needs to be outside + #covariance function k(x) def cov(self, x, y): k = lambda x, y: (self.sigma_k**2)*math.exp((-0.5/self.l_k**2)*np.linalg.norm(x-y)**2) return k(x, y) - #cur_length is needed so that rest of matrix stays Id + #Compute covariance matrix. cur_length is needed so that rest of matrix stays Id def get_cov_matrix(self, C_x, cur_length: int): K = np.identity(len(C_x)) for i in range (0, cur_length, 1): for j in range (0, cur_length, 1): K[i][j]=self.cov(C_x[i], C_x[j]) + #check if covariance matrix is singular - this may happen and is a problem if(np.linalg.det(K) == 0): print("The covariance matrix is singular! It is " + str(K)) print("Current C_x is: " + str(C_x)) - #Maybe change matrix if it is singular? E.g. to scalar*Id, so it's def. not the smallest? - #Would have to deal with that in other methods though - #Important!! K is inverted. But also it's used some way in g?? Unsure what to do - #at least it doesn't crash rn - #add small values to diagonal? -> verfälscht sachen. Müsste relativ zu werten in Matrix angepasst werden - #z.b. 10^-15 oder so K = np.identity(len(C_x))*(cur_length+10) return K - #returns the vector k for a certain input new_x + #returns covariance vector k for a certain input new_x def get_cov_vector(self, C_x, new_x, cur_length: int): #somehow the rest of the function would still work even if x is wrong size, but no idea why / what it does. Better to check if(len(C_x[0]) != len(new_x)): @@ -297,13 +298,11 @@ def create_evidence_set(self, amt_it: int, dim_HP: int): new_x = self.create_random_x() x[i] = new_x #evaluating takes quite some time, especially for min_lv>1 - #y[i] = perform_evaluation_at(data, new_x[0], new_x[1], new_x[2], new_x[3]) - #needs to be casted to int bc otherwise it's automatically float which doesn't work y[i] = self.perform_evaluation_at(x[i]) return x, y - #returns a random x for the given purpose - needs search space + #returns a random x, that fullfills criteria specified in hp_space def create_random_x(self): res = [] for i in range (0, len(self.hp_space)): @@ -321,7 +320,7 @@ def create_random_x(self): new_x = random.uniform(self.hp_space[i][1], self.hp_space[i][2]) elif (self.hp_space[i][0] == "interval_int"): new_x = int(random.randrange(int(self.hp_space[i][1]), int(self.hp_space[i][2]+1))) - #max+1 bc second border value is not included + #max+1 because otherwise second border value is not included else: print("Unknown type of space! Using first value given for index " + str(i)) new_x = self.hp_space[i][1] @@ -329,10 +328,8 @@ def create_random_x(self): return res #checks if array has the element x, returns True if yes. Caution: If x has same length but different dimension to the elements of array this may give unintended results - #maybe later a threshhold can be added if only part of the matrix should be checked def check_if_in_array(self, x, array): if(len(x) != len(array[0])): - #print("not same size") return False for i in range(0, len(array), 1): if((x == array[i]).all()): @@ -363,26 +360,26 @@ def get_new_beta_and_l(self, cur_beta: float, cur_amt_x, cur_x, C_x, C_y, amt_tr #making upper bounds dependable on current values so bounds are never too low beta_h = cur_beta+(100*2**amt_tries) l_h = self.l_k+(50*2**amt_tries) - #0 if rd(x) is in ev set C_x, constant otherwise (5) - #penalty p(x) erhöhen wenn gleicher Wert rauskommt. z.B. immer +50 oder *2 oder anders exponentiell + #penalty p(x) is 0 if rd(x) is in ev set C_x, constant otherwise. Is increased on each try to find new beta and l p = lambda x: self.check_if_in_array(self.round_x(x), C_x)*(200*2**amt_tries) #gets the x value for certain l (z[1]) and beta (z[0]) new_x = lambda z: self.acq_x(z[0], z[1], C_x, C_y, cur_amt_x) #for g: x[0] is \beta+d\beta, x[1] is l. Also d\beta = \beta+d\beta-\beta g = lambda x: (x[0]-cur_beta)+np.linalg.norm(cur_x-new_x(x))+p(new_x(x)) bounds_g = ((cur_beta, beta_h), (self.l_k, l_h)) - #due to numerical inaccuracies a matrix might become singular with new beta and l - #(even though it wouldn't be) mathematically - how to deal with that?? + #due to numerical inaccuracies (or a bug?) a matrix might become singular with new beta and l result = minimize(g, [cur_beta, self.l_k], method='L-BFGS-B', bounds=bounds_g).x #result is in the form [new beta, new l] return result + """returns the bounds in which a new x may lie, as well as an x0 where to start the search for a new x. + Bounds are derived from hp_space, x0 is the smallest value within the bounds. + """ def get_bounds_and_x0(self, offset = 0): bounds = [] x0 = np.zeros(len(self.hp_space)) for i in range(0, len(self.hp_space)): new = [0, 1] - #should one be able to enter just one value for "list"?? if(len(self.hp_space[i])<2): print("please enter at least 1 value for hp_space list. Using default values [0, 1]") elif(self.hp_space[i][0] == "list"): @@ -412,38 +409,27 @@ def get_bounds_and_x0(self, offset = 0): x0[i] = new[0] return bounds, x0 - #acquire new x for l, beta, C_x, cur_amt_x, using GP-UCB + #acquire new x for current l, beta, C_x, cur_amt_x, using GP-UCB def acq_x(self, beta: float, l: float, C_x, C_y, cur_amt_x): - #old_l=self.l_k self.l_k=l K_matr = self.get_cov_matrix(C_x, cur_amt_x) if(np.linalg.det(K_matr) == 0): print("Covariance Matrix is singular!") - #return a value that's bullshit? like [0, 0, 0, 0] (min lv is >=1) - #print(K_matr) mu = lambda x: self.get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(C_y)) sigma_sqrd = lambda x: self.cov(x, x)-self.get_cov_vector(C_x, x, cur_amt_x).dot(np.linalg.inv(K_matr).dot(self.get_cov_vector(C_x, x, cur_amt_x))) #takes sqrt of abs(sigma_sqrd) bc otherwise fmin gives an error - might be an overall warning sign tho sigma = lambda x: math.sqrt(abs(sigma_sqrd(x))) alpha = lambda x: mu(x)+(math.sqrt(abs(beta)))*sigma(x) - #trying out stuff: - #alpha = lambda x: mu(x) - #negates alpha bc maximum has to be found + #negates alpha because maximum needs to be found alpha_neg = lambda x: -alpha(x) - #x2 = np.linspace(0, 20, 21) - #y2 = [alpha_neg(np.array([x_i, 0, 1, 0])) for x_i in x2] - #plt.plot(x2, y2) bounds_and_x0 = self.get_bounds_and_x0() #bounds search space to vicinity of useful values bounds_an = bounds_and_x0[0] - #problem: Nelder-Mead (same as fmin) cannot handle bounds -> use L-BFGS-B for now + #problem: Nelder-Mead (same as fmin) cannot handle bounds -> use L-BFGS-B x0 = bounds_and_x0[1] new_x=minimize(alpha_neg, x0, method='L-BFGS-B', bounds=bounds_an).x - #plt.title(str(new_x)) - #plt.show() - #print("new x: " + str(new_x) + " with function value: " + str(alpha(new_x))) return new_x - #needs search space + #rounds x so that x_rd fulfills criteria stated in hp_space def round_x(self, x): if len(x) < len(self.hp_space): print("Input too short! Returning default values rd(0) for missing values") @@ -451,8 +437,6 @@ def round_x(self, x): x.append(0) if len(x) > len(self.hp_space): print("Input too long! Cropping") - #rounds the values of new_x to values usable as HPs - the question is what kind of rounding makes sense - #e.g if lambda should be bounded or more values should be rounded to 0 new_x_rd = [] for i in range (0, len(self.hp_space)): new_x = 1 @@ -498,8 +482,9 @@ class Optimize_Classification: Options are: moons, blobs, circles, classification, gaussian_quantiles, digits, iris, breast_cancer, wine. Can also take in a dataset directly. :param data: Optional. If no data set is given here data is created by name, using "moons" as default - :param max_lv & max_evals: Parameters of classification and class_dim_wise that are not otimized + :param max_lv: Parameter of classification and class_dim_wise that are not otimized but can be specified here + :param max_evals: see max_lv """ def __init__(self, data_name: str = "moons", data = None, samples: int = 500, dimension: int = 2, labels: int = 6, max_lv: int = 3, max_evals: int = 256): self.samples = samples @@ -538,11 +523,11 @@ def create_data(self, name: str = "moons"): return data """Perform classification with certain HP-input and return evaluation - Parameters are: - :params[0]: lambd_exp: used to calculate lambda = 10**(-lambd_exp) - :params[1]: massl - :params[2]: min_lv - :params[3]: ove_vs_others + :param params: array containing all parameters for classification + params[0]: lambd_exp: used to calculate lambda = 10**(-lambd_exp) + params[1]: massl + params[2]: min_lv + params[3]: ove_vs_others """ def pea_classification(self, params): if(len(params)<4): @@ -556,24 +541,23 @@ def pea_classification(self, params): return evaluation["Percentage correct"] """Perform dimension wise classification with certain HP-input and return evaluation - Parameters are: - :params[0]: lambd_exp: used to calculate lambda = 10**(-lambd_exp) - :params[1]: massl - :params[2]: min_lv - :params[3]: ove_vs_others / error calculator. - case 0: ovo=false, ec=none; - case1: ovo=true, ec=none; - case2: ovo=true, ec=error_calculator - :params[4]: margin - :params[5]: rebalancing - :params[6]: use_relative_surplus - :params[7]: max_evaluations + :param params: array containing all parameters for dim wise classification + params[0]: lambd_exp: used to calculate lambda = 10**(-lambd_exp) + params[1]: massl + params[2]: min_lv + params[3]: ove_vs_others / error calculator. + case 0: ovo=false, ec=none; + case1: ovo=true, ec=none; + case2: ovo=true, ec=error_calculator + params[4]: margin + params[5]: rebalancing + params[6]: use_relative_surplus + params[7]: max_evaluations """ def pea_classification_dimension_wise(self, params): if(len(params)<4): print("too little parameters for pea_classification. Returning 0.0") return 0.0 - #lambd, massl, min_lv, one_vs_others / error calc, margin(float 0-1), rebalancing (bool), use_relative_surplus (bool), max_evaluations (int) params = [float(params[0]), int(params[1]), int(params[2]), int(params[3]), float(params[4]), int(params[5]), int(params[6]), int(params[7])] cur_data = self.data.copy() error_calculator=sparseSpACE.ErrorCalculator.ErrorCalculatorSingleDimMisclassificationGlobal() @@ -587,7 +571,6 @@ def pea_classification_dimension_wise(self, params): classification = deml.Classification(cur_data, split_percentage=0.8, split_evenly=True, shuffle_data=False) classification.perform_classification_dimension_wise(masslumping=params[1], lambd=10**(-params[0]), minimum_level=params[2], maximum_level=self.max_lv, one_vs_others=ovo, error_calculator = ec, margin = params[4], rebalancing = params[5], use_relative_surplus = params[6], max_evaluations = params[7], print_metrics=False) evaluation = classification.evaluate() - #wenn Zeit mit reingerechnet werden soll o.ä. in dieser Funktion return evaluation["Percentage correct"]