DNAS stuff + some experimental hooks

CMakeLists.txt

@@ -26,6 +26,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 # 2 This line for cluster
 #SET (CMAKE_CXX_FLAGS                "-std=gnu++17 -Wall -O3 -funroll-loops  -msse3 -fno-omit-frame-pointer -D_GLIBCXX_DEBUG")
 
+# SET (CMAKE_CXX_FLAGS                " -Wall -O3 -funroll-loops -msse3 -fsanitize=address")
 SET (CMAKE_CXX_FLAGS                " -Wall -O3 -funroll-loops -msse3")
 #SET (CMAKE_CXX_FLAGS                " -Wall -O1 -funroll-loops -msse3 -g -fsanitize=address -fno-omit-frame-pointer -shared-libasan -DGLIBCXX_DEBUG")
 #SET (CMAKE_CXX_FLAGS_DEBUG          "${CMAKE_CXX_FLAGS} -g")

README.md

@@ -50,7 +50,6 @@ Update the code format before start a pull request with:
 ~/exact $ sh scripts/util/format.sh
 ```
 
-
 You may also want to have graphviz installed so you can generate images of the evolved neural networks.  EXACT/EXALT/EXAMM will write out evolved genomes in a .gv (graphviz) format for this. For example, can generate a pdf from a gv file (assuming graphviz is installed with):
 
 ```

common/files.hxx

@@ -1,6 +1,8 @@
 #ifndef EXACT_BOINC_COMMON_HXX
 #define EXACT_BOINC_COMMON_HXX
 
+#include <stdint.h>
+
 #include <stdexcept>
 using std::runtime_error;
 

common/log.cxx

@@ -79,11 +79,11 @@ int8_t Log::parse_level_from_string(string level) {
 void Log::initialize(const vector<string>& arguments) {
     // TODO: should read these from the CommandLine (to be created)
 
-    string std_message_level_str, file_message_level_str;
+    string std_message_level_str = "INFO", file_message_level_str = "NONE";
 
-    get_argument(arguments, "--std_message_level", true, std_message_level_str);
-    get_argument(arguments, "--file_message_level", true, file_message_level_str);
-    get_argument(arguments, "--output_directory", true, output_directory);
+    get_argument(arguments, "--std_message_level", false, std_message_level_str);
+    get_argument(arguments, "--file_message_level", false, file_message_level_str);
+    get_argument(arguments, "--output_directory", false, output_directory);
 
     std_message_level = parse_level_from_string(std_message_level_str);
     file_message_level = parse_level_from_string(file_message_level_str);

common/process_arguments.cxx

@@ -189,10 +189,10 @@ void get_train_validation_data(
     time_series_sets->export_training_series(time_offset, train_inputs, train_outputs);
     time_series_sets->export_test_series(time_offset, validation_inputs, validation_outputs);
 
-    int32_t sequence_length = 0;
-    if (get_argument(arguments, "--train_sequence_length", false, sequence_length)) {
-        Log::info("Slicing input training data with time sequence length: %d\n", sequence_length);
-        slice_input_data(train_inputs, train_outputs, sequence_length);
+    int32_t train_sequence_length = 0;
+    if (get_argument(arguments, "--train_sequence_length", false, train_sequence_length)) {
+        Log::info("Slicing input training data with time sequence length: %d\n", train_sequence_length);
+        slice_input_data(train_inputs, train_outputs, train_sequence_length);
     }
 
     int32_t validation_sequence_length = 0;

examm/examm.cxx

@@ -96,8 +96,9 @@ void EXAMM::generate_log() {
         Log::info("Generating fitness log\n");
         mkpath(output_directory.c_str(), 0777);
         log_file = new ofstream(output_directory + "/" + "fitness_log.csv");
-        (*log_file) << "Inserted Genomes, Total BP Epochs, Time, Best Val. MAE, Best Val. MSE, Enabled Nodes, Enabled "
-                       "Edges, Enabled Rec. Edges";
+        (*log_file
+        ) << "Inserted Genomes,Total BP Epochs,Time,Best Val. MAE,Best Val. MSE,Enabled Nodes,Enabled"
+             "Edges,Enabled Rec. Edges,Val. MSE,Pre-Insert MSE,Genome Inserted,Trainable Parameters,Island Id";
         (*log_file) << speciation_strategy->get_strategy_information_headers();
         (*log_file) << endl;
 
@@ -152,7 +153,7 @@ void EXAMM::update_op_log_statistics(RNN_Genome* genome, int32_t insert_position
     }
 }
 
-void EXAMM::update_log() {
+void EXAMM::update_log(RNN_Genome* genome) {
     if (log_file != NULL) {
         // make sure the log file is still good
         if (!log_file->good()) {
@@ -183,17 +184,26 @@ void EXAMM::update_log() {
             }
             (*op_log_file) << endl;
         }
+
         RNN_Genome* best_genome = get_best_genome();
         if (best_genome == NULL) {
             best_genome = speciation_strategy->get_global_best_genome();
         }
+
         std::chrono::time_point<std::chrono::system_clock> currentClock = std::chrono::system_clock::now();
         long milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(currentClock - startClock).count();
         (*log_file) << speciation_strategy->get_evaluated_genomes() << "," << total_bp_epochs << "," << milliseconds
                     << "," << best_genome->best_validation_mae << "," << best_genome->best_validation_mse << ","
                     << best_genome->get_enabled_node_count() << "," << best_genome->get_enabled_edge_count() << ","
-                    << best_genome->get_enabled_recurrent_edge_count()
-                    << speciation_strategy->get_strategy_information_values() << endl;
+                    << best_genome->get_enabled_recurrent_edge_count() << "," << genome->best_validation_mse << ","
+                    << pre_insert_best_mse << "," << (int32_t) (last_genome_inserted ? 1 : 0) << ","
+                    << genome->get_number_weights() << "," << genome->get_generation_id()
+                    << speciation_strategy->get_strategy_information_values(genome) << endl;
+        Log::info(
+            "mse: %f node count: %d edge count: %d rec edges: %d\n", best_genome->best_validation_mse,
+            best_genome->get_enabled_node_count(), best_genome->get_enabled_edge_count(),
+            best_genome->get_enabled_recurrent_edge_count()
+        );
     }
 }
 
@@ -246,6 +256,8 @@ bool EXAMM::insert_genome(RNN_Genome* genome) {
 
     // updates EXAMM's mapping of which genomes have been generated by what
     genome->update_generation_map(generated_from_map);
+    pre_insert_best_mse = this->get_best_fitness();
+
     int32_t insert_position = speciation_strategy->insert_genome(genome);
     Log::info("insert to speciation strategy complete, at position: %d\n", insert_position);
 
@@ -264,12 +276,14 @@ bool EXAMM::insert_genome(RNN_Genome* genome) {
     }
     Log::info("save genome complete\n");
 
+    last_genome_inserted = insert_position >= 0;
+
     speciation_strategy->print();
-    Log::info("printed speciation strategy!\n\n");
 
     update_op_log_statistics(genome, insert_position);
-    update_log();
-    return insert_position >= 0;
+    update_log(genome);
+
+    return last_genome_inserted;
 }
 
 // write function to save genomes to file
@@ -303,7 +317,6 @@ RNN_Genome* EXAMM::generate_genome() {
     RNN_Genome* genome = speciation_strategy->generate_genome(rng_0_1, generator, mutate_function, crossover_function);
 
     genome_property->set_genome_properties(genome);
-    // if (!epigenetic_weights) genome->initialize_randomly();
 
     // this is just a sanity check, can most likely comment out (checking to see
     // if all the paramemters are sane)
@@ -314,7 +327,7 @@ RNN_Genome* EXAMM::generate_genome() {
     return genome;
 }
 
-int32_t EXAMM::get_random_node_type() {
+node_t EXAMM::get_random_node_type() {
     return possible_node_types[rng_0_1(generator) * possible_node_types.size()];
 }
 
@@ -354,7 +367,8 @@ void EXAMM::mutate(int32_t max_mutations, RNN_Genome* g) {
 
         g->assign_reachability();
         double rng = rng_0_1(generator) * total;
-        int32_t new_node_type = get_random_node_type();
+        node_t new_node_type = get_random_node_type();
+        Log::info("%d %d\n", new_node_type, NODE_TYPES.size());
         string node_type_str = NODE_TYPES[new_node_type];
         Log::debug("rng: %lf, total: %lf, new node type: %d (%s)\n", rng, total, new_node_type, node_type_str.c_str());
 
@@ -536,8 +550,8 @@ void EXAMM::attempt_edge_insert(
             exit(1);
 
             return;
-        } else if (child_edges[i]->get_input_innovation_number() == edge->get_input_innovation_number() &&
-                   child_edges[i]->get_output_innovation_number() == edge->get_output_innovation_number()) {
+        } else if (child_edges[i]->get_input_innovation_number() == edge->get_input_innovation_number()
+                   && child_edges[i]->get_output_innovation_number() == edge->get_output_innovation_number()) {
             Log::debug(
                 "Not inserting edge in crossover operation as there was already an edge with the same input and output "
                 "innovation numbers!\n"
@@ -620,10 +634,10 @@ void EXAMM::attempt_recurrent_edge_insert(
             exit(1);
 
             return;
-        } else if (child_recurrent_edges[i]->get_input_innovation_number() ==
-                       recurrent_edge->get_input_innovation_number() &&
-                   child_recurrent_edges[i]->get_output_innovation_number() ==
-                       recurrent_edge->get_output_innovation_number()) {
+        } else if (child_recurrent_edges[i]->get_input_innovation_number()
+                       == recurrent_edge->get_input_innovation_number()
+                   && child_recurrent_edges[i]->get_output_innovation_number()
+                          == recurrent_edge->get_output_innovation_number()) {
             Log::debug(
                 "Not inserting recurrent_edge in crossover operation as there was already an recurrent_edge with the "
                 "same input and output innovation numbers!\n"

examm/examm.hxx

@@ -63,8 +63,8 @@ class EXAMM {
     double split_node_rate;
     double merge_node_rate;
 
-    vector<int32_t> possible_node_types = {SIMPLE_NODE, JORDAN_NODE, ELMAN_NODE, UGRNN_NODE,
-                                           MGU_NODE,    GRU_NODE,    DELTA_NODE, LSTM_NODE};
+    vector<node_t> possible_node_types = {SIMPLE_NODE, JORDAN_NODE, ELMAN_NODE, UGRNN_NODE,
+                                          MGU_NODE,    GRU_NODE,    DELTA_NODE, LSTM_NODE};
 
     vector<string> op_log_ordering;
     map<string, int32_t> inserted_counts;
@@ -73,6 +73,8 @@ class EXAMM {
     string output_directory;
     ofstream* log_file;
     ofstream* op_log_file;
+    double pre_insert_best_mse = 1000000;
+    bool last_genome_inserted = false;
 
     std::chrono::time_point<std::chrono::system_clock> startClock;
 
@@ -89,13 +91,13 @@ class EXAMM {
     ~EXAMM();
 
     void print();
-    void update_log();
+    void update_log(RNN_Genome* genome);
 
     void set_possible_node_types(vector<string> possible_node_type_strings);
 
     uniform_int_distribution<int32_t> get_recurrent_depth_dist();
 
-    int32_t get_random_node_type();
+    node_t get_random_node_type();
 
     RNN_Genome* generate_genome();
     bool insert_genome(RNN_Genome* genome);

examm/island_speciation_strategy.cxx

@@ -100,12 +100,12 @@ int32_t IslandSpeciationStrategy::get_evaluated_genomes() const {
     return evaluated_genomes;
 }
 
-RNN_Genome* IslandSpeciationStrategy::get_best_genome() {
+RNN_Genome* IslandSpeciationStrategy::get_best_genome() const {
     // the global_best_genome is updated every time a genome is inserted
     return global_best_genome;
 }
 
-RNN_Genome* IslandSpeciationStrategy::get_worst_genome() {
+RNN_Genome* IslandSpeciationStrategy::get_worst_genome() const {
     int32_t worst_genome_island = -1;
     double worst_fitness = -EXAMM_MAX_DOUBLE;
 
@@ -126,7 +126,7 @@ RNN_Genome* IslandSpeciationStrategy::get_worst_genome() {
     }
 }
 
-double IslandSpeciationStrategy::get_best_fitness() {
+double IslandSpeciationStrategy::get_best_fitness() const {
     RNN_Genome* best_genome = get_best_genome();
     if (best_genome == NULL) {
         return EXAMM_MAX_DOUBLE;
@@ -135,7 +135,7 @@ double IslandSpeciationStrategy::get_best_fitness() {
     }
 }
 
-double IslandSpeciationStrategy::get_worst_fitness() {
+double IslandSpeciationStrategy::get_worst_fitness() const {
     RNN_Genome* worst_genome = get_worst_genome();
     if (worst_genome == NULL) {
         return EXAMM_MAX_DOUBLE;
@@ -376,11 +376,15 @@ RNN_Genome* IslandSpeciationStrategy::generate_genome(
         Log::info("Island %d: new genome is still null, regenerating\n", generation_island);
         new_genome = generate_genome(rng_0_1, generator, mutate, crossover);
     }
+
     generated_genomes++;
     new_genome->set_generation_id(generated_genomes);
     islands[generation_island]->set_latest_generation_id(generated_genomes);
     new_genome->set_group_id(generation_island);
 
+    pair<double, double> perf = {this->get_best_fitness(), this->get_worst_fitness()};
+    genome_performance.emplace(new_genome->generation_id, perf);
+
     if (current_island->is_initializing()) {
         RNN_Genome* genome_copy = new_genome->copy();
         Log::debug("inserting genome copy!\n");
@@ -461,6 +465,7 @@ void IslandSpeciationStrategy::print(string indent) const {
  */
 string IslandSpeciationStrategy::get_strategy_information_headers() const {
     string info_header = "";
+    info_header.append(",mse_min_pre,mse_max_pre,mse_min_post,mse_max_post");
     for (int32_t i = 0; i < (int32_t) islands.size(); i++) {
         info_header.append(",");
         info_header.append("Island_");
@@ -477,8 +482,22 @@ string IslandSpeciationStrategy::get_strategy_information_headers() const {
 /**
  * Gets speciation strategy information values for logs
  */
-string IslandSpeciationStrategy::get_strategy_information_values() const {
+string IslandSpeciationStrategy::get_strategy_information_values(RNN_Genome* genome) const {
     string info_value = "";
+
+    auto& [min_mse_pre, max_mse_pre] = genome_performance.at(genome->generation_id);
+    info_value.append(",");
+    info_value.append(to_string(min_mse_pre));
+    info_value.append(",");
+    info_value.append(to_string(max_mse_pre));
+
+    float min_mse_post = this->get_best_fitness();
+    float max_mse_post = this->get_worst_fitness();
+    info_value.append(",");
+    info_value.append(to_string(min_mse_post));
+    info_value.append(",");
+    info_value.append(to_string(max_mse_post));
+
     for (int32_t i = 0; i < (int32_t) islands.size(); i++) {
         double best_fitness = islands[i]->get_best_fitness();
         double worst_fitness = islands[i]->get_worst_fitness();
@@ -584,8 +603,8 @@ void IslandSpeciationStrategy::set_erased_islands_status() {
 RNN_Genome* IslandSpeciationStrategy::get_seed_genome() {
     return seed_genome;
 }
-// write a save entire population function with an input saving function
 
+// write a save entire population function with an input saving function
 void IslandSpeciationStrategy::save_entire_population(string output_path) {
     for (int32_t i = 0; i < (int32_t) islands.size(); i++) {
         islands[i]->save_population(output_path);

examm/island_speciation_strategy.hxx

@@ -62,6 +62,12 @@ class IslandSpeciationStrategy : public SpeciationStrategy {
     vector<Island*> islands;
     RNN_Genome* global_best_genome;
 
+    ofstream* island_log_file;
+
+    // Maps genome number to a pair representing (worst island mse, best island mse) at
+    // the time of genome generation.
+    unordered_map<int32_t, pair<double, double>> genome_performance;
+
     // Transfer learning class properties:
 
     bool transfer_learning;
@@ -114,25 +120,25 @@ class IslandSpeciationStrategy : public SpeciationStrategy {
      * Gets the fitness of the best genome of all the islands
      * \return the best fitness over all islands
      */
-    double get_best_fitness();
+    double get_best_fitness() const;
 
     /**
      * Gets the fitness of the worst genome of all the islands
      * \return the worst fitness over all islands
      */
-    double get_worst_fitness();
+    double get_worst_fitness() const;
 
     /**
      * Gets the best genome of all the islands
      * \return the best genome of all islands or NULL if no genomes have yet been inserted
      */
-    RNN_Genome* get_best_genome();
+    RNN_Genome* get_best_genome() const;
 
     /**
      * Gets the the worst genome of all the islands
      * \return the worst genome of all islands or NULL if no genomes have yet been inserted
      */
-    RNN_Genome* get_worst_genome();
+    RNN_Genome* get_worst_genome() const;
 
     /**
      *  \return true if all the islands are full
@@ -207,7 +213,7 @@ class IslandSpeciationStrategy : public SpeciationStrategy {
     /**
      * Gets speciation strategy information values for logs
      */
-    string get_strategy_information_values() const;
+    string get_strategy_information_values(RNN_Genome* genome) const;
 
     /**
      * Island repopulation through two random parents from two seperate islands,