add ability to specify delimiter for CSV inputs

CMakeLists.txt

@@ -26,11 +26,12 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 # 2 This line for cluster
 #SET (CMAKE_CXX_FLAGS                "-std=gnu++17 -Wall -O3 -funroll-loops  -msse3 -fno-omit-frame-pointer -D_GLIBCXX_DEBUG")
 
-SET (CMAKE_CXX_FLAGS                " -Wall -O3 -funroll-loops -msse3")
+# SET (CMAKE_CXX_FLAGS                " -Wall -O3 -funroll-loops -msse3 -fsanitize=address")
+SET (CMAKE_CXX_FLAGS                " -Wall -march=native -O3")
 #SET (CMAKE_CXX_FLAGS                " -Wall -O1 -funroll-loops -msse3 -g -fsanitize=address -fno-omit-frame-pointer -shared-libasan -DGLIBCXX_DEBUG")
 #SET (CMAKE_CXX_FLAGS_DEBUG          "${CMAKE_CXX_FLAGS} -g")
 #SET (CMAKE_CXX_FLAGS_MINSIZEREL     "${CMAKE_CXX_FLAGS} -Os -DNDEBUG")
-SET (CMAKE_CXX_FLAGS_RELEASE        "${CMAKE_CXX_FLAGS} -O4 -funroll-loops -DNDEBUG")
+SET (CMAKE_CXX_FLAGS_RELEASE        "${CMAKE_CXX_FLAGS} -DNDEBUG")
 
 #SET (CMAKE_SHARED_LINKER_FLAGS      " -Wall -O1 -funroll-loops -msse3 -g -fsanitize=address -fno-omit-frame-pointer -shared-libasan -DGLIBCXX_DEBUG")
 

README.md

@@ -50,7 +50,6 @@ Update the code format before start a pull request with:
 ~/exact $ sh scripts/util/format.sh
 ```
 
-
 You may also want to have graphviz installed so you can generate images of the evolved neural networks.  EXACT/EXALT/EXAMM will write out evolved genomes in a .gv (graphviz) format for this. For example, can generate a pdf from a gv file (assuming graphviz is installed with):
 
 ```

common/files.hxx

@@ -1,6 +1,8 @@
 #ifndef EXACT_BOINC_COMMON_HXX
 #define EXACT_BOINC_COMMON_HXX
 
+#include <stdint.h>
+
 #include <stdexcept>
 using std::runtime_error;
 

common/log.cxx

@@ -79,11 +79,11 @@ int8_t Log::parse_level_from_string(string level) {
 void Log::initialize(const vector<string>& arguments) {
     // TODO: should read these from the CommandLine (to be created)
 
-    string std_message_level_str, file_message_level_str;
+    string std_message_level_str = "INFO", file_message_level_str = "NONE";
 
-    get_argument(arguments, "--std_message_level", true, std_message_level_str);
-    get_argument(arguments, "--file_message_level", true, file_message_level_str);
-    get_argument(arguments, "--output_directory", true, output_directory);
+    get_argument(arguments, "--std_message_level", false, std_message_level_str);
+    get_argument(arguments, "--file_message_level", false, file_message_level_str);
+    get_argument(arguments, "--output_directory", false, output_directory);
 
     std_message_level = parse_level_from_string(std_message_level_str);
     file_message_level = parse_level_from_string(file_message_level_str);

common/process_arguments.cxx

@@ -1,3 +1,4 @@
+#include <memory>
 #include <string>
 using std::string;
 
@@ -120,12 +121,15 @@ IslandSpeciationStrategy* generate_island_speciation_strategy_from_arguments(
     get_argument(arguments, "--seed_stirs", false, seed_stirs);
     bool start_filled = argument_exists(arguments, "--start_filled");
     bool tl_epigenetic_weights = argument_exists(arguments, "--tl_epigenetic_weights");
+    unique_ptr<AnnealingPolicy> annealing_policy = AnnealingPolicy::from_arguments(arguments);
+    string output_directory = "";
+    get_argument(arguments, "--output_directory", false, output_directory);
 
     IslandSpeciationStrategy* island_strategy = new IslandSpeciationStrategy(
-        number_islands, island_size, mutation_rate, intra_island_co_rate, inter_island_co_rate, seed_genome,
+        number_islands, island_size, mutation_rate, intra_island_co_rate, inter_island_co_rate, output_directory, seed_genome,
         island_ranking_method, repopulation_method, extinction_event_generation_number, num_mutations,
         islands_to_exterminate, max_genomes, repeat_extinction, start_filled, transfer_learning,
-        transfer_learning_version, seed_stirs, tl_epigenetic_weights
+        transfer_learning_version, tl_epigenetic_weights, annealing_policy
     );
 
     return island_strategy;
@@ -189,10 +193,10 @@ void get_train_validation_data(
     time_series_sets->export_training_series(time_offset, train_inputs, train_outputs);
     time_series_sets->export_test_series(time_offset, validation_inputs, validation_outputs);
 
-    int32_t sequence_length = 0;
-    if (get_argument(arguments, "--train_sequence_length", false, sequence_length)) {
-        Log::info("Slicing input training data with time sequence length: %d\n", sequence_length);
-        slice_input_data(train_inputs, train_outputs, sequence_length);
+    int32_t train_sequence_length = 0;
+    if (get_argument(arguments, "--train_sequence_length", false, train_sequence_length)) {
+        Log::info("Slicing input training data with time sequence length: %d\n", train_sequence_length);
+        slice_input_data(train_inputs, train_outputs, train_sequence_length);
     }
 
     int32_t validation_sequence_length = 0;

examm/CMakeLists.txt

@@ -1 +1 @@
-add_library(examm_strategy examm.cxx  species.cxx island.cxx island_speciation_strategy.cxx species.cxx neat_speciation_strategy.cxx)
+add_library(examm_strategy examm.cxx species.cxx island.cxx island_speciation_strategy.cxx species.cxx neat_speciation_strategy.cxx annealing.cxx)

examm/annealing.cxx

@@ -0,0 +1,86 @@
+#include "annealing.hxx"
+
+#include <cmath>
+#include <memory>
+
+#include "common/arguments.hxx"
+#include "common/log.hxx"
+
+unique_ptr<AnnealingPolicy> AnnealingPolicy::from_arguments(const vector<string>& arguments) {
+    string type;
+    get_argument(arguments, "--annealing_policy", false, type);
+    Log::info("Annealing policy = %s\n", type.c_str());
+    if (type == "linear") {
+        return unique_ptr<AnnealingPolicy>(new LinearAnnealingPolicy(arguments));
+    } else if (type == "inv_exp") {
+        return unique_ptr<AnnealingPolicy>(new InvExpAnnealingPolicy(arguments));
+    } else if (type == "sin") {
+        return unique_ptr<AnnealingPolicy>(new SinAnnealingPolicy(arguments));
+    } else {
+        Log::info("Using default annealing policy\n");
+        return make_unique<AnnealingPolicy>();
+    }
+}
+
+double AnnealingPolicy::operator()(int32_t genome_number) {
+    return 0.0;
+}
+
+LinearAnnealingPolicy::LinearAnnealingPolicy(
+    double start_value, double end_value, int32_t start_genomes, int32_t interp_genomes
+)
+    : start_value(start_value), end_value(end_value), start_genomes(start_genomes), interp_genomes(interp_genomes) {
+}
+
+LinearAnnealingPolicy::LinearAnnealingPolicy(const vector<string>& arguments) {
+    get_argument(arguments, "--linear_start_value", true, start_value);
+    get_argument(arguments, "--linear_end_value", true, end_value);
+    get_argument(arguments, "--linear_start_genomes", true, start_genomes);
+    get_argument(arguments, "--linear_interp_genomes", true, interp_genomes);
+}
+
+double LinearAnnealingPolicy::operator()(int32_t genome_number) {
+    if (genome_number <= start_genomes) {
+        return start_value;
+    } else if (genome_number <= interp_genomes + start_genomes) {
+        double weight = (double) (genome_number - (interp_genomes + start_genomes)) / (double) interp_genomes;
+        return weight * end_value + (1 - weight) * start_value;
+    } else {
+        return end_value;
+    }
+}
+
+InvExpAnnealingPolicy::InvExpAnnealingPolicy(double decay_factor) : decay_factor(decay_factor) {
+}
+InvExpAnnealingPolicy::InvExpAnnealingPolicy(const vector<string>& arguments) {
+    get_argument(arguments, "--exp_decay_factor", true, decay_factor);
+}
+
+double InvExpAnnealingPolicy::operator()(int32_t genome_number) {
+    return std::pow(1. + genome_number, -decay_factor);
+}
+
+SinAnnealingPolicy::SinAnnealingPolicy(double period, double min_p, double max_p)
+    : period(period), min_p(min_p), max_p(max_p) {
+    if (min_p > max_p) {
+        std::swap(min_p, max_p);
+    }
+
+    if (min_p > 1.0 || min_p < 0.0) {
+        throw "Invalid min_p supplied to SinAnnealingPolicyConstructor";
+    }
+    if (max_p > 1.0 || max_p < 0.0) {
+        throw "Invalid max_p supplied to SinAnnealingPolicyConstructor";
+    }
+}
+SinAnnealingPolicy::SinAnnealingPolicy(const vector<string>& arguments) {
+    get_argument(arguments, "--sin_min_p", true, min_p);
+    get_argument(arguments, "--sin_max_p", true, max_p);
+    get_argument(arguments, "--sin_period", true, period);
+}
+
+double SinAnnealingPolicy::operator()(int32_t genome_number) {
+    double range = max_p - min_p;
+
+    return (max_p + min_p) / 2. + range / 2. * std::sin(2. * M_PI * genome_number / period);
+}

examm/annealing.hxx

@@ -0,0 +1,65 @@
+#include <stdint.h>
+
+#include <memory>
+using std::unique_ptr;
+
+#include <string>
+using std::string;
+
+#include <vector>
+using std::vector;
+
+struct AnnealingPolicy {
+    static unique_ptr<AnnealingPolicy> from_arguments(const vector<string>& arguments);
+
+    /**
+     * Compute the probability to be used during genome insertion.
+     * This represents the probability of inserting the genome, even if it
+     * has a fitness value that is worse than the worst member in the population.
+     */
+    virtual double operator()(int32_t genome_number);
+};
+
+/**
+ * Interpolate between two values for a set number of genomes.
+ * The `start_value` will be returned for `start_genomes`,
+ * then a linear interpolation of `start_value` and `end_value` for
+ * `interp_genomes`. Then, `end_value` is given indefinitely.
+ */
+class LinearAnnealingPolicy : public AnnealingPolicy {
+    double start_value, end_value;
+    int32_t start_genomes, interp_genomes;
+
+   public:
+    LinearAnnealingPolicy(double start_value, double end_value, int32_t start_genomes, int32_t interp_genomes);
+    LinearAnnealingPolicy(const vector<string>& arguments);
+
+    virtual double operator()(int32_t genome_number);
+};
+
+/**
+ * Calculates p by simply computing `genome_number^(-decay_factor).
+ **/
+class InvExpAnnealingPolicy : public AnnealingPolicy {
+    double decay_factor;
+
+   public:
+    InvExpAnnealingPolicy(double decay_factor);
+    InvExpAnnealingPolicy(const vector<string>& arguments);
+
+    virtual double operator()(int32_t genome_number);
+};
+
+/**
+ * Computes `p` as a value falling on a sinusoidal curve with the supplied period.
+ * a `min_p` and a `max_p` specify the range of the curve.
+ **/
+class SinAnnealingPolicy : public AnnealingPolicy {
+    double period, min_p, max_p;
+
+   public:
+    SinAnnealingPolicy(double period, double min_p, double max_p);
+    SinAnnealingPolicy(const vector<string>& arguments);
+
+    virtual double operator()(int32_t genome_number);
+};

examm/examm.cxx

@@ -96,8 +96,10 @@ void EXAMM::generate_log() {
         Log::info("Generating fitness log\n");
         mkpath(output_directory.c_str(), 0777);
         log_file = new ofstream(output_directory + "/" + "fitness_log.csv");
-        (*log_file) << "Inserted Genomes, Total BP Epochs, Time, Best Val. MAE, Best Val. MSE, Enabled Nodes, Enabled "
-                       "Edges, Enabled Rec. Edges";
+        (*log_file
+        ) << "Inserted Genomes,Total BP Epochs,Time,Best Val. MAE,Best Val. MSE,Trainable Parameters,Enabled "
+             "Nodes,Enabled"
+             "Edges,Enabled Rec. Edges,Val. MSE,Pre-Insert MSE,Genome Inserted,Genome Trainable Parameters,Island Id";
         (*log_file) << speciation_strategy->get_strategy_information_headers();
         (*log_file) << endl;
 
@@ -152,7 +154,7 @@ void EXAMM::update_op_log_statistics(RNN_Genome* genome, int32_t insert_position
     }
 }
 
-void EXAMM::update_log() {
+void EXAMM::update_log(RNN_Genome* genome) {
     if (log_file != NULL) {
         // make sure the log file is still good
         if (!log_file->good()) {
@@ -183,17 +185,27 @@ void EXAMM::update_log() {
             }
             (*op_log_file) << endl;
         }
+
         RNN_Genome* best_genome = get_best_genome();
         if (best_genome == NULL) {
             best_genome = speciation_strategy->get_global_best_genome();
         }
+
         std::chrono::time_point<std::chrono::system_clock> currentClock = std::chrono::system_clock::now();
         long milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(currentClock - startClock).count();
         (*log_file) << speciation_strategy->get_evaluated_genomes() << "," << total_bp_epochs << "," << milliseconds
                     << "," << best_genome->best_validation_mae << "," << best_genome->best_validation_mse << ","
-                    << best_genome->get_enabled_node_count() << "," << best_genome->get_enabled_edge_count() << ","
-                    << best_genome->get_enabled_recurrent_edge_count()
-                    << speciation_strategy->get_strategy_information_values() << endl;
+                    << best_genome->get_number_weights() << "," << best_genome->get_enabled_node_count() << ","
+                    << best_genome->get_enabled_edge_count() << "," << best_genome->get_enabled_recurrent_edge_count()
+                    << "," << genome->best_validation_mse << "," << pre_insert_best_mse << ","
+                    << (int32_t) (last_genome_inserted ? 1 : 0) << "," << genome->get_number_weights() << ","
+                    << genome->get_generation_id() << speciation_strategy->get_strategy_information_values(genome)
+                    << endl;
+        Log::info(
+            "mse: %f node count: %d edge count: %d rec edges: %d\n", best_genome->best_validation_mse,
+            best_genome->get_enabled_node_count(), best_genome->get_enabled_edge_count(),
+            best_genome->get_enabled_recurrent_edge_count()
+        );
     }
 }
 
@@ -246,15 +258,17 @@ bool EXAMM::insert_genome(RNN_Genome* genome) {
 
     // updates EXAMM's mapping of which genomes have been generated by what
     genome->update_generation_map(generated_from_map);
+    pre_insert_best_mse = this->get_best_fitness();
+
     int32_t insert_position = speciation_strategy->insert_genome(genome);
     Log::info("insert to speciation strategy complete, at position: %d\n", insert_position);
 
     // write this genome to disk if it was a new best found genome
     if (save_genome_option.compare("all_best_genomes") == 0) {
-        Log::info("save genome option compared, save genome option size: %d!\n", save_genome_option.size());
-        for (int i = 0; i < 20 && i < save_genome_option.size(); i++) {
-            cout << "save_genome_option[" << i << "]: " << save_genome_option[i] << endl;
-        }
+        // Log::info("save genome option compared, save genome option size: %d!\n", save_genome_option.size());
+        // for (int i = 0; i < 20 && i < save_genome_option.size(); i++) {
+        //     cout << "save_genome_option[" << i << "]: " << save_genome_option[i] << endl;
+        // }
 
         if (insert_position == 0) {
             Log::info("saving genome!");
@@ -264,16 +278,26 @@ bool EXAMM::insert_genome(RNN_Genome* genome) {
     }
     Log::info("save genome complete\n");
 
+    last_genome_inserted = insert_position >= 0;
+
     speciation_strategy->print();
-    Log::info("printed speciation strategy!\n\n");
 
     update_op_log_statistics(genome, insert_position);
-    update_log();
-    return insert_position >= 0;
+    update_log(genome);
+
+    return last_genome_inserted;
 }
 
 // write function to save genomes to file
 void EXAMM::save_genome(RNN_Genome* genome, string genome_name = "rnn_genome") {
+    if (genome->get_fitness() != EXAMM_MAX_DOUBLE) {
+        // need to set the weights for non-initial genomes so we
+        // can generate a proper graphviz file
+        vector<double> best_parameters = genome->get_best_parameters();
+        genome->set_weights(best_parameters);
+        Log::info("set genome parameters to best\n");
+    }
+
     genome->write_graphviz(output_directory + "/" + genome_name + "_" + to_string(genome->get_generation_id()) + ".gv");
     ofstream equations_filestream(
         output_directory + "/" + genome_name + "_" + to_string(genome->get_generation_id()) + ".txt"
@@ -314,7 +338,7 @@ RNN_Genome* EXAMM::generate_genome() {
     return genome;
 }
 
-int32_t EXAMM::get_random_node_type() {
+node_t EXAMM::get_random_node_type() {
     return possible_node_types[rng_0_1(generator) * possible_node_types.size()];
 }
 
@@ -354,7 +378,8 @@ void EXAMM::mutate(int32_t max_mutations, RNN_Genome* g) {
 
         g->assign_reachability();
         double rng = rng_0_1(generator) * total;
-        int32_t new_node_type = get_random_node_type();
+        node_t new_node_type = get_random_node_type();
+        Log::info("%d %d\n", new_node_type, NODE_TYPES.size());
         string node_type_str = NODE_TYPES[new_node_type];
         Log::debug("rng: %lf, total: %lf, new node type: %d (%s)\n", rng, total, new_node_type, node_type_str.c_str());