diff --git a/cpp/.gitignore b/cpp/.gitignore index d73b742ba..3dc8c5baf 100644 --- a/cpp/.gitignore +++ b/cpp/.gitignore @@ -5,4 +5,3 @@ build/ .settings/ .vscode/ cmake-build-debug/ -cmake*/ \ No newline at end of file diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index b4b8c39bf..5045f6eb1 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -108,10 +108,10 @@ if (NOT WIN32) endif () endif () -option(BUILD_TEST "Build tests" ON) +option(BUILD_TEST "Build tests" OFF) message("cmake using: BUILD_TEST=${BUILD_TEST}") -option(ENABLE_ANTLR4 "Enable ANTLR4 runtime" ON) +option(ENABLE_ANTLR4 "Enable ANTLR4 runtime" OFF) message("cmake using: ENABLE_ANTLR4=${ENABLE_ANTLR4}") option(ENABLE_SNAPPY "Enable Google Snappy compression" ON) diff --git a/cpp/README.md b/cpp/README.md index dced6c8c1..a98792eaa 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -75,7 +75,7 @@ Modify the Toolchain File `cmake/ToolChain.cmake`, define the following variable In the `cpp/` directory, run the following commands to create the build directory and start the compilation: ``` mkdir build && cd build -cmake .. -DToolChian=ON +cmake .. -DToolChain=ON make ``` diff --git a/cpp/cmake/ToolChain.cmake b/cpp/cmake/ToolChain.cmake new file mode 100644 index 000000000..f52be2934 --- /dev/null +++ b/cpp/cmake/ToolChain.cmake @@ -0,0 +1,36 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] + +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR aarch64) + +## Modify +set(CMAKE_C_COMPILER /home/tsfile/dev/gcc-linaro-5.5.0-2017.10-x86_64_arm-linux-gnueabi/bin/arm-linux-gnueabi-gcc CACHE STRING "Path to the C compiler" FORCE) +set(CMAKE_CXX_COMPILER /home/colin/dev/gcc-linaro-5.5.0-2017.10-x86_64_arm-linux-gnueabi/bin/arm-linux-gnueabi-g++ CACHE STRING "Path to the C++ compiler" FORCE) +message(STATUS "using cxx compiler ${CMAKE_CXX_COMPILER}") +message(STATUS "using c compiler ${CMAKE_C_COMPILER}") +## Modify +set(CMAKE_FIND_ROOT_PATH /home/tsfile/dev/gcc-linaro-5.5.0-2017.10-x86_64_arm-linux-gnueabi) + + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE BOTH) + +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") \ No newline at end of file diff --git a/cpp/examples/CMakeLists.txt b/cpp/examples/CMakeLists.txt index ebe6c66c8..69c7b130e 100644 --- a/cpp/examples/CMakeLists.txt +++ b/cpp/examples/CMakeLists.txt @@ -34,11 +34,11 @@ set(SDK_LIB_DIR_DEBUG ${PROJECT_SOURCE_DIR}/../build/Debug/lib) message("SDK_LIB_DIR_DEBUG: ${SDK_LIB_DIR_DEBUG}") include_directories(${PROJECT_SOURCE_DIR}/../third_party/antlr4-cpp-runtime-4/runtime/src) -set(BUILD_TYPE "Release") include_directories(${SDK_INCLUDE_DIR}) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g") -set(CMAKE_CXX_FLAGS_DEBUG" ${CMAKE_CXX_FLAGS} -O0 -g") +set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Build type" FORCE) +set(CMAKE_C_FLAGS_RELEASE "-O2 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG" CACHE STRING "" FORCE) add_subdirectory(cpp_examples) add_subdirectory(c_examples) diff --git a/cpp/examples/c_examples/demo_read.c b/cpp/examples/c_examples/demo_read.c index 05cc8620a..4c2f06fd6 100644 --- a/cpp/examples/c_examples/demo_read.c +++ b/cpp/examples/c_examples/demo_read.c @@ -29,7 +29,7 @@ ERRNO read_tsfile() { char* table_name = "table1"; // Create tsfile reader with specify tsfile's path - TsFileReader reader = tsfile_reader_new("test_c.tsfile", &code); + TsFileReader reader = tsfile_reader_new("test_c_normal.tsfile", &code); HANDLE_ERROR(code); ResultSet ret = tsfile_query_table( diff --git a/cpp/examples/c_examples/demo_write.c b/cpp/examples/c_examples/demo_write.c index 925e3af8c..98e0b4b97 100644 --- a/cpp/examples/c_examples/demo_write.c +++ b/cpp/examples/c_examples/demo_write.c @@ -21,13 +21,67 @@ #include #include #include +#include #include "c_examples.h" +/* 计时用：单调时钟，返回毫秒 */ +static inline int64_t now_monotonic_ms(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (int64_t)ts.tv_sec * 1000LL + (int64_t)ts.tv_nsec / 1000000LL; +} + +int64_t get_timestamp_int64(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + + return (int64_t)ts.tv_sec * 1000000000LL + + (int64_t)ts.tv_nsec; +} + +static uint32_t g_rng_state = 0xA5A5A5A5u; + +static inline void rng_seed(uint32_t seed) { + /* 避免 0 种子导致退化 */ + g_rng_state = (seed == 0) ? 0xA5A5A5A5u : seed; +} + +static inline uint32_t xorshift32_u32(void) { + uint32_t x = g_rng_state; + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + g_rng_state = x; + return x; +} + +/* 返回 [center - range, center + range] 的均匀分布 int32 */ +static inline int32_t rng_int32_around(int32_t center, int32_t range) { + /* 2*range+1 个离散值：例如 center=25, range=5 -> [20..30] */ + uint32_t span = (uint32_t)(2 * range + 1); + uint32_t r = xorshift32_u32(); + + /* 为了避免取模偏差，用 rejection sampling */ + uint32_t limit = UINT32_MAX - (UINT32_MAX % span); + while (r >= limit) { + r = xorshift32_u32(); + } + + int32_t offset = (int32_t)(r % span) - range; + return center + offset; +} // This example shows you how to write tsfile. ERRNO write_tsfile() { ERRNO code = 0; - code = set_global_compression(TS_COMPRESSION_LZ4); + int64_t t_total_start = now_monotonic_ms(); + remove("test_c.tsfile"); + // Create a file with specify path to write tsfile. + WriteFile file = write_file_new("test_c.tsfile", &code); + HANDLE_ERROR(code); + code = set_global_compression(TS_COMPRESSION_GZIP); if (code != RET_OK) { return code; } @@ -40,50 +94,94 @@ ERRNO write_tsfile() { // Create table schema to describe a table in a tsfile. TableSchema table_schema; table_schema.table_name = strdup(table_name); - table_schema.column_num = 3; + table_schema.column_num = 8; table_schema.column_schemas = - (ColumnSchema*)malloc(sizeof(ColumnSchema) * 3); + (ColumnSchema*)malloc(sizeof(ColumnSchema) * 9); table_schema.column_schemas[0] = - (ColumnSchema){.column_name = strdup("id1"), - .data_type = TS_DATATYPE_STRING, - .column_category = TAG}; + (ColumnSchema){.column_name = strdup("a1"), + .data_type = TS_DATATYPE_INT32, + .column_category = FIELD}; table_schema.column_schemas[1] = - (ColumnSchema){.column_name = strdup("id2"), - .data_type = TS_DATATYPE_STRING, - .column_category = TAG}; + (ColumnSchema){.column_name = strdup("a2"), + .data_type = TS_DATATYPE_INT32, + .column_category = FIELD}; table_schema.column_schemas[2] = - (ColumnSchema){.column_name = strdup("s1"), + (ColumnSchema){.column_name = strdup("a3"), + .data_type = TS_DATATYPE_INT32, + .column_category = FIELD}; + table_schema.column_schemas[3] = + (ColumnSchema){.column_name = strdup("a4"), + .data_type = TS_DATATYPE_INT32, + .column_category = FIELD}; + table_schema.column_schemas[4] = + (ColumnSchema){.column_name = strdup("a5"), + .data_type = TS_DATATYPE_INT32, + .column_category = FIELD}; + table_schema.column_schemas[5] = + (ColumnSchema){.column_name = strdup("a6"), + .data_type = TS_DATATYPE_INT32, + .column_category = FIELD}; + table_schema.column_schemas[6] = + (ColumnSchema){.column_name = strdup("a7"), + .data_type = TS_DATATYPE_INT32, + .column_category = FIELD}; + table_schema.column_schemas[7] = + (ColumnSchema){.column_name = strdup("a8"), .data_type = TS_DATATYPE_INT32, .column_category = FIELD}; - - remove("test_c.tsfile"); - // Create a file with specify path to write tsfile. - WriteFile file = write_file_new("test_c.tsfile", &code); - HANDLE_ERROR(code); // Create tsfile writer with specify table schema. TsFileWriter writer = tsfile_writer_new(file, &table_schema, &code); HANDLE_ERROR(code); // Create tablet to insert data. - Tablet tablet = - tablet_new((char*[]){"id1", "id2", "s1"}, - (TSDataType[]){TS_DATATYPE_STRING, TS_DATATYPE_STRING, - TS_DATATYPE_INT32}, - 3, 5); - - for (int row = 0; row < 5; row++) { - Timestamp timestamp = row; - tablet_add_timestamp(tablet, row, timestamp); - tablet_add_value_by_name_string_with_len( - tablet, row, "id1", "id_field_1", strlen("id_field_1")); - tablet_add_value_by_name_string_with_len( - tablet, row, "id2", "id_field_2", strlen("id_field_2")); - tablet_add_value_by_name_int32_t(tablet, row, "s1", row); - } + int tablet_num = 1000; + int max_len = 1000; + Tablet tablet = tablet_new( + (char*[]){"a1", "a2", "a3", "a4", "a5", "a6", "a7", "a8"}, + (TSDataType[]){ TS_DATATYPE_INT32, TS_DATATYPE_INT32, + TS_DATATYPE_INT32, TS_DATATYPE_INT32, TS_DATATYPE_INT32, + TS_DATATYPE_INT32, TS_DATATYPE_INT32, TS_DATATYPE_INT32}, + 8, max_len); + printf("\ndebug %d.\n", 4); + + int64_t sum_fill_ms = 0; /* tablet 填充耗时总和 */ + int64_t sum_write_ms = 0; /* writer_write 耗时总和 */ + + for (uint32_t tablet_idx = 0; tablet_idx < tablet_num; tablet_idx++) { + int64_t t_fill_start = now_monotonic_ms(); + for (uint32_t row = 0; row < max_len; row++) { + Timestamp timestamp = get_timestamp_int64(); + tablet_add_timestamp(tablet, row, timestamp); + tablet_add_value_by_name_int32_t(tablet, row, "a1", + rng_int32_around(25, 5)); + tablet_add_value_by_name_int32_t(tablet, row, "a2", + rng_int32_around(25, 5)); + tablet_add_value_by_name_int32_t(tablet, row, "a3", + rng_int32_around(25, 5)); + tablet_add_value_by_name_int32_t(tablet, row, "a4", + rng_int32_around(25, 5)); + tablet_add_value_by_name_int32_t(tablet, row, "a5", + rng_int32_around(25, 5)); + tablet_add_value_by_name_int32_t(tablet, row, "a6", + rng_int32_around(25, 5)); + tablet_add_value_by_name_int32_t(tablet, row, "a7", + rng_int32_around(25, 5)); + tablet_add_value_by_name_int32_t(tablet, row, "a8", + rng_int32_around(25, 5)); + } + int64_t t_fill_end = now_monotonic_ms(); + int64_t fill_ms = t_fill_end - t_fill_start; + sum_fill_ms += fill_ms; + - // Write tablet data. - HANDLE_ERROR(tsfile_writer_write(writer, tablet)); + int64_t t_write_start = now_monotonic_ms(); + HANDLE_ERROR(tsfile_writer_write(writer, tablet)); + int64_t t_write_end = now_monotonic_ms(); + + int64_t write_ms = t_write_end - t_write_start; + sum_write_ms += write_ms; + } // Free tablet. free_tablet(&tablet); @@ -92,10 +190,24 @@ ERRNO write_tsfile() { free_table_schema(table_schema); // Close writer. + int64_t t_close_start = now_monotonic_ms(); HANDLE_ERROR(tsfile_writer_close(writer)); - - // Close write file after closing writer. free_write_file(&file); + int64_t t_close_end = now_monotonic_ms(); + int64_t close_ms = t_close_end - t_close_start; + + int64_t t_total_end = now_monotonic_ms(); + int64_t total_ms = t_total_end - t_total_start; + + printf("\n[PERF][SUMMARY]\n"); + printf("[PERF][SUMMARY] tablet_num=%d max_len=%d rows_total=%lld\n", + tablet_num, max_len, (long long)tablet_num * (long long)max_len); + printf("[PERF][SUMMARY] fill_total_ms=%lld avg_fill_ms=%.3f\n", + (long long)sum_fill_ms, (double)sum_fill_ms / (double)tablet_num); + printf("[PERF][SUMMARY] write_total_ms=%lld avg_write_ms=%.3f\n", + (long long)sum_write_ms, (double)sum_write_ms / (double)tablet_num); + printf("[PERF][SUMMARY] close_ms=%lld\n", (long long)close_ms); + printf("[PERF][SUMMARY] total_ms=%lld\n\n", (long long)total_ms); return 0; } \ No newline at end of file diff --git a/cpp/examples/examples.cc b/cpp/examples/examples.cc index edbd819a0..151d5cdcc 100644 --- a/cpp/examples/examples.cc +++ b/cpp/examples/examples.cc @@ -23,11 +23,11 @@ int main() { // C++ examples // std::cout << "begin write and read tsfile by cpp" << std::endl; - demo_write(); - demo_read(); - std::cout << "begin write and read tsfile by c" << std::endl; +// demo_write(); +// demo_read(); +// std::cout << "begin write and read tsfile by c" << std::endl; // C examples write_tsfile(); - read_tsfile(); +// read_tsfile(); return 0; } \ No newline at end of file diff --git a/cpp/src/common/db_common.h b/cpp/src/common/db_common.h index 485a0c104..8c637c3da 100644 --- a/cpp/src/common/db_common.h +++ b/cpp/src/common/db_common.h @@ -20,7 +20,9 @@ #ifndef COMMON_DB_COMMON_H #define COMMON_DB_COMMON_H +#include #include +#include #include #include "common/allocator/my_string.h" @@ -94,7 +96,20 @@ enum CompressionType : uint8_t { extern const char* s_data_type_names[8]; extern const char* s_encoding_names[12]; extern const char* s_compression_names[8]; +} // namespace common +#if defined(__GLIBCXX__) && (__GNUC__ < 7) +namespace std { +template <> +struct hash { + size_t operator()(common::TSDataType v) const noexcept { + return static_cast(static_cast(v)); + } +}; +} // namespace std +#endif + +namespace common { FORCE_INLINE const char* get_data_type_name(TSDataType type) { ASSERT(type >= BOOLEAN && type <= STRING); return s_data_type_names[type]; @@ -142,40 +157,38 @@ FORCE_INLINE common::TSDataType GetDataTypeFromTemplateType() { } template -FORCE_INLINE std::unordered_set -GetDataTypesFromTemplateType() { - return {common::INVALID_DATATYPE}; +FORCE_INLINE bool TypeMatch(common::TSDataType dt) { + return dt == common::INVALID_DATATYPE; } template <> -FORCE_INLINE std::unordered_set -GetDataTypesFromTemplateType() { - return {common::BOOLEAN}; +FORCE_INLINE bool TypeMatch(common::TSDataType dt) { + return dt == common::BOOLEAN; } + template <> -FORCE_INLINE std::unordered_set -GetDataTypesFromTemplateType() { - return {common::INT32, common::DATE, common::INT64}; +FORCE_INLINE bool TypeMatch(common::TSDataType dt) { + return dt == common::INT32 || dt == common::DATE || dt == common::INT64; } + template <> -FORCE_INLINE std::unordered_set -GetDataTypesFromTemplateType() { - return {common::INT64, TIMESTAMP}; +FORCE_INLINE bool TypeMatch(common::TSDataType dt) { + return dt == common::INT64 || dt == common::TIMESTAMP; } + template <> -FORCE_INLINE std::unordered_set -GetDataTypesFromTemplateType() { - return {common::FLOAT, common::DOUBLE}; +FORCE_INLINE bool TypeMatch(common::TSDataType dt) { + return dt == common::FLOAT || dt == common::DOUBLE; } + template <> -FORCE_INLINE std::unordered_set -GetDataTypesFromTemplateType() { - return {common::DOUBLE}; +FORCE_INLINE bool TypeMatch(common::TSDataType dt) { + return dt == common::DOUBLE; } + template <> -FORCE_INLINE std::unordered_set -GetDataTypesFromTemplateType() { - return {common::STRING, common::TEXT, common::BLOB}; +FORCE_INLINE bool TypeMatch(common::TSDataType dt) { + return dt == common::STRING || dt == common::TEXT || dt == common::BLOB; } FORCE_INLINE size_t get_data_type_size(TSDataType data_type) { diff --git a/cpp/src/common/tablet.cc b/cpp/src/common/tablet.cc index 2a22d78d0..84225473d 100644 --- a/cpp/src/common/tablet.cc +++ b/cpp/src/common/tablet.cc @@ -255,8 +255,7 @@ int Tablet::add_value(uint32_t row_index, uint32_t schema_index, T val) { ret = common::E_OUT_OF_RANGE; } else { const MeasurementSchema& schema = schema_vec_->at(schema_index); - auto dic = GetDataTypesFromTemplateType(); - if (dic.find(schema.data_type_) == dic.end()) { + if (UNLIKELY(!TypeMatch(schema.data_type_))) { return E_TYPE_NOT_MATCH; } process_val(row_index, schema_index, val); @@ -281,27 +280,6 @@ int Tablet::add_value(uint32_t row_index, uint32_t schema_index, std::tm val) { return ret; } -template <> -int Tablet::add_value(uint32_t row_index, uint32_t schema_index, - common::String val) { - if (err_code_ != E_OK) { - return err_code_; - } - int ret = common::E_OK; - if (UNLIKELY(schema_index >= schema_vec_->size())) { - ASSERT(false); - ret = common::E_OUT_OF_RANGE; - } else { - const MeasurementSchema& schema = schema_vec_->at(schema_index); - auto dic = GetDataTypesFromTemplateType(); - if (dic.find(schema.data_type_) == dic.end()) { - return E_TYPE_NOT_MATCH; - } - process_val(row_index, schema_index, val); - } - return ret; -} - template <> int Tablet::add_value(uint32_t row_index, uint32_t schema_index, const char* val) { diff --git a/cpp/src/writer/tsfile_writer.cc b/cpp/src/writer/tsfile_writer.cc index 1f7d954ce..3e92dff62 100644 --- a/cpp/src/writer/tsfile_writer.cc +++ b/cpp/src/writer/tsfile_writer.cc @@ -863,6 +863,15 @@ TsFileWriter::split_tablet_by_device(const Tablet& tablet) { std::vector, int>> result; std::shared_ptr last_device_id = std::make_shared("last_device_id"); + if (tablet.id_column_indexes_.empty()) { + result.emplace_back(std::move(last_device_id), 0); + std::vector id_array; + id_array.push_back(new std::string(tablet.insert_target_name_)); + auto res = std::make_shared(id_array); + result.emplace_back(std::move(res), tablet.get_cur_row_size()); + return result; + } + for (uint32_t i = 0; i < tablet.get_cur_row_size(); i++) { std::shared_ptr cur_device_id(tablet.get_device_id(i)); if (*cur_device_id != *last_device_id) {