Refactor data element

src/Column.hxx

@@ -7,27 +7,41 @@ namespace tablator {
 
 class Column {
 public:
+    // These constexprs are used by HDF5-support code.
     static constexpr char const *COL_ARRAY_SIZE = "array_size";
     static constexpr char const *COL_FIELD_PROPERTIES = "field_properties";
     static constexpr char const *COL_NAME = "name";
     static constexpr char const *COL_TYPE = "type";
+    static constexpr char const *COL_DYNAMIC_ARRAY_FLAG = "dynamic_array_flag";
 
-    Column(const std::string &Name, const Data_Type &Type, const size_t &Array_size)
-            : Column(Name, Type, Array_size, Field_Properties()) {}
+    Column(const std::string &name, const Data_Type &type, const size_t &array_size,
+           const Field_Properties &field_properties, bool dynamic_array_flag)
+            : name_(name),
+              type_(type),
+              array_size_(array_size),
+              field_properties_(field_properties),
+              dynamic_array_flag_(dynamic_array_flag) {}
 
-    Column(const std::string &Name, const Data_Type &Type, const size_t &Array_size,
-           const Field_Properties &Field_properties)
-            : name_(Name),
-              type_(Type),
-              array_size_(Array_size),
-              field_properties_(Field_properties) {}
+    Column(const std::string &name, const Data_Type &type, const size_t &array_size,
+           const Field_Properties &field_properties)
+            : Column(name, type, array_size, field_properties,
+                     ((type == Data_Type::CHAR) ||
+                      (array_size == std::numeric_limits<size_t>::max()))) {}
 
-    inline size_t get_data_size() const {
-        return tablator::data_size(type_) * array_size_;
-    }
+    Column(const std::string &name, const Data_Type &type, const size_t &array_size,
+           bool dynamic_array_flag)
+            : Column(name, type, array_size, Field_Properties(), dynamic_array_flag) {}
+
+
+    Column(const std::string &name, const Data_Type &type, const size_t &array_size)
+            : Column(name, type, array_size, Field_Properties()) {}
 
-    // deprecated
-    inline size_t data_size() const { return get_data_size(); }
+
+    Column(const std::string &name, const Data_Type &type,
+           const Field_Properties &field_properties)
+            : Column(name, type, 1, field_properties) {}
+
+    Column(const std::string &name, const Data_Type &type) : Column(name, type, 1) {}
 
     // accessors
     inline const std::string &get_name() const { return name_; }
@@ -63,7 +77,12 @@ public:
     inline ATTRIBUTES &get_field_property_attributes() {
         return get_field_properties().get_attributes();
     }
+    inline bool get_dynamic_array_flag() const { return dynamic_array_flag_; }
+    inline void set_dynamic_array_flag(bool b) { dynamic_array_flag_ = b; }
 
+    inline size_t get_data_size() const {
+        return tablator::data_size(type_) * array_size_;
+    }
 
 private:
     std::string name_;
@@ -72,6 +91,7 @@ private:
     // Actual array_size for fixed-length arrays; maximum array_size otherwise.
     size_t array_size_;
     Field_Properties field_properties_;
+    bool dynamic_array_flag_;
 };
 
 

src/Common.hxx

@@ -64,6 +64,7 @@ static const std::string PARAMREF("PARAMref");
 static const std::string QUERY_STATUS("QUERY_STATUS");
 static const std::string REF("ref");
 static const std::string RESOURCE("RESOURCE");
+static const std::string STREAM("STREAM");
 static const std::string TABLE("TABLE");
 static const std::string TABLEDATA("TABLEDATA");
 static const std::string TIMESYS("TIMESYS");

src/Data_Details.hxx

@@ -0,0 +1,72 @@
+#pragma once
+
+#include "Common.hxx"
+#include "Field_Framework.hxx"
+#include "Row.hxx"
+
+namespace tablator {
+
+class Data_Details {
+public:
+    Data_Details(size_t row_size, size_t num_rows = 0) : row_size_(row_size) {
+        init(num_rows);
+    }
+
+    Data_Details(const Field_Framework &field_framework, size_t num_rows = 0)
+            : Data_Details(field_framework.get_row_size(), num_rows) {}
+
+
+    void append_row(const Row &row) {
+        assert(row.get_data().size() == get_row_size());
+        data_.reserve(data_.size() + row.get_data().size());
+        data_.insert(data_.end(), row.get_data().begin(), row.get_data().end());
+    }
+
+
+    void append_rows(const Data_Details &other) {
+        assert(other.get_row_size() == get_row_size());
+
+        data_.reserve(data_.size() + other.get_data().size());
+        data_.insert(data_.end(), other.get_data().begin(), other.get_data().end());
+    }
+
+    void adjust_num_rows(const size_t new_num_rows) {
+        data_.resize(new_num_rows * get_row_size());
+    }
+
+    void reserve_rows(const size_t &new_num_rows) {
+        data_.reserve(get_row_size() * new_num_rows);
+    }
+
+    // accessors
+
+    size_t get_data_size() const { return data_.size(); }
+
+    size_t get_num_rows() const {
+        if (get_row_size() == 0) {
+            return 0;
+        }
+        return get_data_size() / get_row_size();
+    };
+
+    inline size_t get_row_size() const { return row_size_; }
+
+    inline const std::vector<uint8_t> &get_data() const { return data_; }
+
+    // Non-const to support append_rows().
+    inline std::vector<uint8_t> &get_data() { return data_; }
+
+    inline void set_data(const std::vector<uint8_t> &d) { data_ = d; }
+
+
+private:
+    void init(const size_t &new_num_rows) {
+	  reserve_rows(new_num_rows);
+    }
+
+    // Can't be const because of append_rows().
+    std::vector<uint8_t> data_;
+    size_t row_size_;
+};
+
+}  // namespace tablator

src/Data_Element.hxx

@@ -1,37 +1,67 @@
 #pragma once
 
-#include <boost/property_tree/ptree.hpp>
-#include <boost/property_tree/xml_parser.hpp>
-
 #include "Column.hxx"
-
-// JTODO: Create struct to hold commonly used (columns, offsets) pair?
+#include "Data_Details.hxx"
+#include "Field_Framework.hxx"
 
 namespace tablator {
 
 class Data_Element {
 public:
-    Data_Element(const std::vector<Column> &columns, const std::vector<size_t> &offsets,
-                 const std::vector<uint8_t> &data)
-            : columns_(columns), offsets_(offsets), data_(data) {}
+    Data_Element(const Field_Framework &field_framework,
+                 const Data_Details &data_details)
+            : field_framework_(field_framework), data_details_(data_details) {}
+
+    Data_Element(const Field_Framework &field_framework, size_t num_rows = 0)
+            : Data_Element(field_framework, Data_Details(field_framework, num_rows)) {}
 
     // accessors
-    inline const std::vector<Column> &get_columns() const { return columns_; }
-    inline std::vector<Column> &get_columns() { return columns_; }
+    const Field_Framework &get_field_framework() const { return field_framework_; }
+
+    Field_Framework &get_field_framework() { return field_framework_; }
+
+
+    const Data_Details &get_data_details() const { return data_details_; }
+
+    Data_Details &get_data_details() { return data_details_; }
 
-    inline const std::vector<size_t> &get_offsets() const { return offsets_; }
-    inline std::vector<size_t> &get_offsets() { return offsets_; }
+    inline const std::vector<Column> &get_columns() const {
+        return field_framework_.get_columns();
+    }
 
+    // Non-const to allow query_server to modify field_properties.
+    inline std::vector<Column> &get_columns() { return field_framework_.get_columns(); }
 
-    inline const std::vector<uint8_t> &get_data() const { return data_; }
-    inline std::vector<uint8_t> &get_data() { return data_; }
+    inline const std::vector<size_t> &get_offsets() const {
+        return field_framework_.get_offsets();
+    }
 
-    inline void set_data(const std::vector<uint8_t> &d) { data_ = d; }
+    size_t get_row_size() const { return field_framework_.get_row_size(); }
+    size_t get_num_rows() const { return data_details_.get_num_rows(); }
+
+    void adjust_num_rows(const size_t &new_num_rows) {
+        data_details_.adjust_num_rows(new_num_rows);
+    }
+
+    void reserve_rows(const size_t &new_num_rows) {
+        get_data_details().reserve_rows(new_num_rows);
+    }
+
+    inline const std::vector<uint8_t> &get_data() const {
+        return data_details_.get_data();
+    }
+
+    // Non-const to support append_rows().
+    inline std::vector<uint8_t> &get_data() { return data_details_.get_data(); }
+
+    inline void set_data(const std::vector<uint8_t> &d) { data_details_.set_data(d); }
 
 private:
-    std::vector<Column> columns_;
-    std::vector<size_t> offsets_ = {0};
-    std::vector<uint8_t> data_;
+    // Non-const to allow query_server to update column's field_properties.
+    Field_Framework field_framework_;
+
+    // Non-const because of append_rows().
+    Data_Details data_details_;
 };
 
 }  // namespace tablator

src/Field_Framework.hxx

@@ -0,0 +1,95 @@
+#pragma once
+
+#include "Column.hxx"
+#include "Utils/Null_Utils.hxx"
+
+namespace tablator {
+
+class Field_Framework {
+public:
+    Field_Framework(const std::vector<Column> &incoming_columns,
+                    bool got_null_bitfields_column = false)
+            : offsets_({0}) {
+        if ((got_null_bitfields_column && incoming_columns.size() == 1) ||
+            incoming_columns.empty()) {
+            throw std::runtime_error(
+                    "Field_Framework constructor: <visible_columns> must be "
+                    "non-empty.");
+        }
+        // JTODO avoid copies
+        if (!got_null_bitfields_column) {
+            const size_t null_flags_size = bits_to_bytes(incoming_columns.size() + 1);
+            append_column(null_bitfield_flags_name, Data_Type::UINT8_LE,
+                          null_flags_size,
+                          Field_Properties::Builder()
+                                  .add_description(null_bitfield_flags_description)
+                                  .build());
+        }
+
+        for (auto &col : incoming_columns) {
+            append_column(col);
+        }
+    }
+
+    // accessors
+
+    inline size_t get_row_size() const {
+        if (offsets_.empty()) {
+            throw std::runtime_error("<offsets> is empty");
+        }
+        return offsets_.back();
+    }
+
+    inline const std::vector<Column> &get_columns() const { return columns_; }
+    inline std::vector<Column> &get_columns() { return columns_; }
+
+    inline const std::vector<size_t> &get_offsets() const { return offsets_; }
+    inline std::vector<size_t> &get_offsets() { return offsets_; }
+
+private:
+    void append_column(const Column &column) {
+        columns_.emplace_back(column);
+        size_t old_row_size = *offsets_.rbegin();
+        size_t new_row_size = old_row_size + columns_.rbegin()->get_data_size();
+        offsets_.push_back(new_row_size);
+    }
+
+    inline void append_column(const std::string &name, const Data_Type &type,
+                              const size_t &array_size,
+                              const Field_Properties &field_properties,
+                              bool dynamic_array_flag) {
+        append_column(
+                Column(name, type, array_size, field_properties, dynamic_array_flag));
+    }
+
+    inline void append_column(const std::string &name, const Data_Type &type,
+                              const size_t &array_size,
+                              const Field_Properties &field_properties) {
+        append_column(Column(name, type, array_size, field_properties));
+    }
+
+    inline void append_column(const std::string &name, const Data_Type &type,
+                              const size_t &array_size, bool dynamic_array_flag) {
+        append_column(Column(name, type, array_size, dynamic_array_flag));
+    }
+
+    inline void append_column(const std::string &name, const Data_Type &type,
+                              const size_t &size) {
+        append_column(Column(name, type, size));
+    }
+
+    inline void append_column(const std::string &name, const Data_Type &type,
+                              const Field_Properties &field_properties) {
+        append_column(Column(name, type, field_properties));
+    }
+
+
+    inline void append_column(const std::string &name, const Data_Type &type) {
+        append_column(Column(name, type));
+    }
+
+    std::vector<Column> columns_;
+    std::vector<size_t> offsets_;
+};
+
+}  // namespace tablator

src/Property.hxx

@@ -21,16 +21,6 @@ public:
     Property(const std::string &Value, const ATTRIBUTES &Attributes)
             : attributes_(Attributes), value_(Value) {}
 
-    // Called internally, directly or otherwise, only by flatten_properties().
-    std::vector<STRING_PAIR> flatten(const std::string &key) const {
-        std::vector<STRING_PAIR> result;
-        result.push_back(std::make_pair(key, value_));
-        for (auto &a : attributes_)
-            result.push_back(
-                    std::make_pair(key + "." + XMLATTR_DOT + a.first, a.second));
-        return result;
-    }
-
     const ATTRIBUTES &get_attributes() const { return attributes_; }
     ATTRIBUTES &get_attributes() { return attributes_; }