diff --git a/.clang-tidy b/.clang-tidy index dd0f6158c..88dab71b9 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -135,6 +135,8 @@ Checks: "*, -readability-uppercase-literal-suffix, -readability-use-anyofallof, -readability-redundant-access-specifiers, + -readability-identifier-naming, + -readability-redundant-string-init, -zirkon-*, " diff --git a/.gitignore b/.gitignore index 976a98274..ef0e48b6b 100755 --- a/.gitignore +++ b/.gitignore @@ -86,3 +86,4 @@ docker/rocky9/dingofs .cache dist/ +.playground diff --git a/CMakeLists.txt b/CMakeLists.txt index abc6ffcfa..bd4065acf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -157,6 +157,9 @@ message("OpenSSL libraries: ${OPENSSL_LIBRARIES}, include dir: ${OPENSSL_INCLUDE find_package(Snappy) message("Using Snappy ${Snappy_VERSION}") +find_package(toml11 REQUIRED) +message("Using toml11 ${toml11_VERSION}") + find_package(fmt REQUIRED) message("Using fmt ${fmt_VERSION}") diff --git a/conf/client copy.conf b/conf/client copy.conf new file mode 100644 index 000000000..989bc59a5 --- /dev/null +++ b/conf/client copy.conf @@ -0,0 +1,202 @@ +##### mdsOpt +# RPC total retry time with MDS +mdsOpt.mdsMaxRetryMS=16000 +# The maximum timeout of RPC communicating with MDS. +# The timeout of exponential backoff cannot exceed this value +mdsOpt.rpcRetryOpt.maxRPCTimeoutMS=2000 +# RPC timeout for once communication with MDS +mdsOpt.rpcRetryOpt.rpcTimeoutMs=500 +# RPC with mds needs to sleep for a period of time before each retry +mdsOpt.rpcRetryOpt.rpcRetryIntervalUS=50000 +# Switch if the number of consecutive retries on the current MDS exceeds the limit. +# The number of failures includes the number of timeout retries +mdsOpt.rpcRetryOpt.maxFailedTimesBeforeChangeAddr=2 +# The normal retry times for trigger wait strategy +mdsOpt.rpcRetryOpt.normalRetryTimesBeforeTriggerWait=3 +# Sleep interval for wait +mdsOpt.rpcRetryOpt.waitSleepMs=1000 +mdsOpt.rpcRetryOpt.addrs=127.0.0.1:6700,127.0.0.1:6701,127.0.0.1:6702 # __ANSIBLE_TEMPLATE__ {{ groups.mds | join_peer(hostvars, "mds_listen_port") }} __ANSIBLE_TEMPLATE__ + + +# +# lease options +# +# client and mds lease time, default is 20s +mds.leaseTimesUs=20000000 +mds.refreshTimesPerLease=5 + +#### metaCacheOpt +# Gets the number of retries for the leader +metaCacheOpt.metacacheGetLeaderRetry=3 +# Need to sleep for a period of time before each get leader retry +metaCacheOpt.metacacheRPCRetryIntervalUS=100000 +# RPC timeout of get leader +metaCacheOpt.metacacheGetLeaderRPCTimeOutMS=1000 + +#### executorOpt +# executorOpt rpc with metaserver +# rpc retry times with metaserver +executorOpt.maxRetry=10 +# internal rpc retry times with metaserver +executorOpt.maxInternalRetry=3 +# Retry sleep time between failed RPCs +executorOpt.retryIntervalUS=100000 +# RPC timeout for communicating with metaserver +executorOpt.rpcTimeoutMS=1000 +# RPC stream idle timeout +executorOpt.rpcStreamIdleTimeoutMS=500 +# The maximum timeout RPC time of the retry request. +# The timeout time will follow the exponential backoff policy. +# Because the timeout occurs when the network is congested, the RPC timeout needs to be increased +executorOpt.maxRPCTimeoutMS=8000 +# Maximum sleep time between retry requests. +# when the network is congested or the metaserver is overloaded, +# it is necessary to increase the sleep time +executorOpt.maxRetrySleepIntervalUS=8000000 +executorOpt.minRetryTimesForceTimeoutBackoff=5 +executorOpt.maxRetryTimesBeforeConsiderSuspend=20 +# batch limit of get inode attr and xattr +executorOpt.batchInodeAttrLimit=10000 + +#### brpc +# close socket after defer.close.second +rpc.defer.close.second=1 +# rpc health check interval in second, 0 or negative value means disable health check +rpc.healthCheckIntervalSec=0 + +#### fuseClient +# TODO(xuchaojie): add unit +fuseClient.listDentryLimit=65536 +fuseClient.downloadMaxRetryTimes=3 + + +# you shoudle enable it when mount one filesystem to multi mountpoints, +# it gurantee the consistent of file after rename, otherwise you should +# disable it for performance. +fuseClient.enableMultiMountPointRename=true +# thread number of listDentry when get summary xattr +fuseClient.listDentryThreads=10 +# default data（s3ChunkInfo/volumeExtent） size in inode, if exceed will eliminate and try to get the merged one +fuseClient.maxDataSize=1024 +# default refresh data interval 30s +fuseClient.refreshDataIntervalSec=30 +fuseClient.warmupThreadsNum=10 + +# when read happen all block of file will be prefetch to block cache +# if no disk cache this option will be ignored +fuseClient.in_time_warmup=false + +fuseClient.bthread_worker_num=0 + + +fs.rpc.listDentryLimit=65536 +fs.deferSync.delay=3 +fs.deferSync.deferDirMtime=false +# } + + + +#### block cache +# { +# block_cache.cache_store: +# cache store type, none, disk or 3fs +# +# block_cache.stage_bandwidth_throttle_enable: +# block will been put to s3 storage directly if disk write bandwidth +# exceed limit. +# +# disk_cache.cache_dir: +# directory for store cache block, multi directories +# and corresponding max size are supported, e.g. "/data1:200;/data2:300" +# +# disk_cache.ioring_iodepth: +# iodepth for io ring (works for both linux io uring and 3fs usrbio) +block_cache.cache_store=disk +block_cache.stage=true +block_cache.stage_bandwidth_throttle_enable=false +block_cache.stage_bandwidth_throttle_mb=10240 +block_cache.logging=true +block_cache.upload_stage_workers=10 +block_cache.upload_stage_queue_size=10000 +block_cache.prefetch_workers=128 +block_cache.prefetch_queue_size=10000 + +disk_cache.cache_dir=/var/run/dingofs # __DINGOADM_TEMPLATE__ /dingofs/client/data/cache __DINGOADM_TEMPLATE__ +disk_cache.cache_size_mb=102400 +disk_cache.free_space_ratio=0.1 +disk_cache.cache_expire_second=259200 +disk_cache.cleanup_expire_interval_millsecond=1000 +disk_cache.drop_page_cache=true +disk_cache.ioring_iodepth=128 +disk_cache.ioring_blksize=1048576 +disk_cache.ioring_prefetch=true + +disk_state.tick_duration_second=60 +disk_state.normal2unstable_io_error_num=3 +disk_state.unstable2normal_io_succ_num=10 +disk_state.unstable2down_second=1800 +disk_state.disk_check_duration_millsecond=3000 +# } + +#### s3 +# this is for test. if s3.fakeS3=true, all data will be discarded +s3.fakeS3=false +# prefetch blocks that disk cache use +s3.prefetchBlocks=1 +# prefetch threads +s3.prefetchExecQueueNum=1 +# start sleep when mem cache use ratio is greater than nearfullRatio, +# sleep time increase follow with mem cache use ratio, baseSleepUs is baseline. +s3.nearfullRatio=70 +s3.baseSleepUs=500 + +# write cache < 8,388,608 (8MB) is not allowed +s3.writeCacheMaxByte=838860800 +s3.readCacheMaxByte=209715200 +# file cache read thread num +s3.readCacheThreads=5 + +s3.verify_SSL=False +s3.region=us-east-1 +s3.maxConnections=32 +s3.connectTimeout=60000 +s3.requestTimeout=10000 +# Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6 +s3.logLevel=4 +s3.logPrefix=/data/logs/dingofs/aws_ # __DINGOADM_TEMPLATE__ /dingofs/client/logs/aws_ __DINGOADM_TEMPLATE__ +# limit all inflight async requests' bytes, |0| means not limited +s3.maxAsyncRequestInflightBytes=104857600 +# throttle +s3.throttle.iopsTotalLimit=0 +s3.throttle.iopsReadLimit=0 +s3.throttle.iopsWriteLimit=0 +s3.throttle.bpsTotalMB=0 +s3.throttle.bpsReadMB=0 +s3.throttle.bpsWriteMB=0 +s3.useVirtualAddressing=false +# The interval between read failures and retries will become larger and larger, +# and when the max is reached, retry will be performed at a fixed time. +s3.maxReadRetryIntervalMs = 1000 +# retry interval +s3.readRetryIntervalMs = 100 + +s3.enableTelemetry=false + +s3.use_crt_client=false +# this only work when use_crt_client is false +s3.use_thread_pool=true +# this only work when use_crt_client is false and use_thread_pool is true +s3.async_thread_num_in_thread_pool=256 + +#### common +client.common.logDir=/data/logs/dingofs # __DINGOADM_TEMPLATE__ /dingofs/client/logs __DINGOADM_TEMPLATE__ +# we have loglevel: {0,3,6,9} +# as the number increases, it becomes more and more detailed +client.loglevel=0 +client.dummyServer.startPort=9000 + + +mds.addr=172.20.61.102:7801 + +### uds +uds.fdCommPath=/var/run # unix domain socket file path \ No newline at end of file diff --git a/conf/client.conf b/conf/client.conf index 3d7d1968c..b4426e9a8 100644 --- a/conf/client.conf +++ b/conf/client.conf @@ -280,7 +280,7 @@ s3.enableTelemetry=false s3.use_crt_client=false # this only work when use_crt_client is false -s3.use_thread_pool=true +s3.use_thread_pool=true # this only work when use_crt_client is false and use_thread_pool is true s3.async_thread_num_in_thread_pool=256 diff --git a/confv2/dingo-cache.gflags b/confv2/dingo-cache.gflags new file mode 100644 index 000000000..0089a6113 --- /dev/null +++ b/confv2/dingo-cache.gflags @@ -0,0 +1,11 @@ +-block_cache_logging=true +-block_cache_stage_bandwidth_throttle_enable=false +-block_cache_stage_bandwidth_throttle_mb=10240 +-disk_cache_cache_expire_s=259200 +-disk_cache_cleanup_expire_interval_ms=1000 +-disk_cache_drop_page_cache=true +-disk_state_tick_duration_s=60 +-disk_state_normal2unstable_io_error_num=3 +-disk_state_unstable2normal_io_succ_num=10 +-disk_state_unstable2down_s=1800 +-disk_state_disk_check_duration_ms=3000 diff --git a/confv2/dingo-cache.template.toml b/confv2/dingo-cache.template.toml new file mode 100644 index 000000000..bce236aa3 --- /dev/null +++ b/confv2/dingo-cache.template.toml @@ -0,0 +1,37 @@ +[cache_group] +group_name = "default" +listen_ip = "127.0.0.1" +listen_port = 9302 +group_weight = 100 +max_range_size_kb = 256 +metadata_filepath = "/var/log/cache_group_meta" +load_members_interval_ms = 1000 + +[block_cache] +logging = true +cache_store = "disk" +stage = true +stage_bandwidth_throttle_enable = false +stage_bandwidth_throttle_mb = 10240 +upload_stage_workers = 10 +upload_stage_queue_size = 10000 +prefetch_workers = 128 +prefetch_queue_size = 10000 + +[block_cache.disk_cache] +cache_dir = [ '/var/run/dingofs' ] +cache_size_mb = 10240 +free_space_ratio = 0.1 +cache_expire_s = 259200 +cleanup_expire_interval_ms = 1000 +drop_page_cache = true +ioring_iodepth = 128 +ioring_blksize = 1048576 +ioring_prefetch = true + +[block_cache.disk_cache.disk_state] +tick_duration_s = 60 +normal2unstable_io_error_num = 3 +unstable2normal_io_succ_num = 10 +unstable2down_s = 1800 +disk_check_duration_ms = 3000 diff --git a/confv2/dingo-fuse.gflags b/confv2/dingo-fuse.gflags new file mode 100644 index 000000000..2c0f15695 --- /dev/null +++ b/confv2/dingo-fuse.gflags @@ -0,0 +1 @@ +-access_lgging=true \ No newline at end of file diff --git a/confv2/dingo-fuse.template.toml b/confv2/dingo-fuse.template.toml new file mode 100644 index 000000000..cf45aa737 --- /dev/null +++ b/confv2/dingo-fuse.template.toml @@ -0,0 +1,138 @@ +# global +[global] +access_logging = true +bthread_worker_num = 0 + +# fuse +[fuse] + +[fuse.conn_info] +want_splice_move=false +want_splice_read=false +want_splice_write=false +want_auto_inval_data=true + +[fuse.file_info] +direct_io=false +keep_cache=true + +# vfs +[vfs] +cto = true +nocto_suffix = "" +max_name_length = 255 +disable_xattr = true + +[vfs.kernel_cache] +attr_timeout_s = 30 +dir_attr_timeout_s = 30 +entry_timeout_s = 30 +dir_entry_timeout_s = 30 + +[vfs.lookup_cache] +negative_timeout_s = 0 +min_uses = 1 +lru_size = 100000 + +[vfs.dir_cache] +lru_size = 5000000 + +[vfs.attr_watcher] +lru_size = 5000000 + +[vfs.throttle] +avg_write_bytes = 0 # write +burst_write_bytes = 0 +burst_write_bytes_s = 180 +avg_write_iops = 0 +burst_write_iops = 0 +burst_write_iops_s = 180 +avg_read_bytes = 0 # read +burst_read_bytes = 0 +burst_read_bytes_s = 180 +avg_read_iops = 0 +burst_read_iops = 0 +burst_read_iops_s = 180 + +# data +[meta] + +[meta.v1.mds] + +[meta.v1.metaserver] + +[meta.v2.mds] +rpc_timeout_ms = 2000 +rpc_retry_times = 3 +client_send_request_retry_times = 3 + +# data +[data] + +[data.background_flush] +interval_ms = 1000 +trigger_force_memory_ratio = 0.90 + +[data.flush_file] +flush_workers = 10 +flush_queue_size = 500 + +[data.flush_chunk] +flush_workers = 10 +flush_queue_size = 500 + +[data.flush_slice] +flush_workers = 10 +flush_queue_size = 500 +stay_in_memory_max_s = 5 + +[data.page] +size = 65536 +total_size_mb = 1024 +use_pool = true + +# rpc +[rpc] +defer_close_s = 1 +health_check_interval_s = 0 + +[rpc.mds] +address = [ "127.0.0.1:6700" ] +retry_total_ms = 16000 +rpc_timeout_ms = 2000 + +[rpc.metaserver] +max_retries = 10 +max_internal_retries = 3 +retry_interval_us = 100000 +rpc_timeout_ms = 1000 +rpc_stream_idle_timeout_ms = 500 +batch_inode_attr_limit = 10000 + +# s3 +[s3] + +[s3.global] +log_level = 4 +log_prefix = "/tmp/" +telemetry_enable = false + +[s3.request] +region = "us-east-1" +use_virtual_addressing = false +verify_ssl = false +max_connections = 32 +connect_timeout_ms = 60000 +request_timeout_ms = 10000 +use_crt_client = false +use_thread_pool = true +async_thread_num = true +max_async_request_inflight_bytes = 0 + +[s3.throttle] +iops_total_limit = 0 +iops_read_limit = 0 +iops_write_limit = 0 +bps_total_mb = 0 +bps_read_mb = 0 +bps_write_mb = 0 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2e6a8560b..542d000eb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +add_subdirectory(options) add_subdirectory(dataaccess) add_subdirectory(utils) add_subdirectory(utils/concurrent) diff --git a/src/cache/blockcache/block_cache.cpp b/src/cache/blockcache/block_cache.cpp index 1783e5b50..88ff093c5 100644 --- a/src/cache/blockcache/block_cache.cpp +++ b/src/cache/blockcache/block_cache.cpp @@ -54,10 +54,11 @@ BlockCacheImpl::BlockCacheImpl(BlockCacheOption option, running_(false), stage_count_(std::make_shared()), throttle_(std::make_unique()) { - if (option.cache_store == "none") { + if (option.cache_store() == "none") { store_ = std::make_shared(); } else { - store_ = std::make_shared(option.disk_cache_options); + store_ = std::make_shared( + option.disk_cache_option()); // FIXME(Wine93) } uploader_ = @@ -70,8 +71,8 @@ BlockCacheImpl::BlockCacheImpl(BlockCacheOption option, Status BlockCacheImpl::Init() { if (!running_.exchange(true)) { throttle_->Start(); - uploader_->Init(option_.upload_stage_workers, - option_.upload_stage_queue_size); + uploader_->Init(option_.upload_stage_workers(), + option_.upload_stage_queue_size()); auto status = store_->Init([this](const BlockKey& key, const std::string& stage_path, BlockContext ctx) { @@ -81,11 +82,11 @@ Status BlockCacheImpl::Init() { return status; } - status = - prefetcher_->Init(option_.prefetch_workers, option_.prefetch_queue_size, - [this](const BlockKey& key, size_t length) { - return DoPrefetch(key, length); - }); + status = prefetcher_->Init(option_.prefetch_workers(), + option_.prefetch_queue_size(), + [this](const BlockKey& key, size_t length) { + return DoPrefetch(key, length); + }); if (!status.ok()) { return status; } @@ -117,7 +118,7 @@ Status BlockCacheImpl::Put(const BlockKey& key, const Block& block, }); auto wait = throttle_->Add(block.size); // stage throttle - if (option_.stage && !wait) { + if (option_.stage() && !wait) { timer.NextPhase(Phase::kStageBlock); status = store_->Stage(key, block, ctx); if (status.ok()) { @@ -217,7 +218,7 @@ bool BlockCacheImpl::IsCached(const BlockKey& key) { } StoreType BlockCacheImpl::GetStoreType() { - if (option_.cache_store == "none") { + if (option_.cache_store() == "none") { return StoreType::kNone; } return StoreType::kDisk; diff --git a/src/cache/blockcache/block_cache_metric.h b/src/cache/blockcache/block_cache_metric.h index 87e7a83a8..d69d96a4c 100644 --- a/src/cache/blockcache/block_cache_metric.h +++ b/src/cache/blockcache/block_cache_metric.h @@ -94,9 +94,9 @@ class BlockCacheMetric { public: BlockCacheMetric(BlockCacheOption option, AuxMember aux_members) : metric_("dingofs_block_cache", aux_members) { - metric_.upload_stage_workers.set_value(option.upload_stage_workers); + metric_.upload_stage_workers.set_value(option.upload_stage_workers()); metric_.upload_stage_queue_capacity.set_value( - option.upload_stage_queue_size); + option.upload_stage_queue_size()); } virtual ~BlockCacheMetric() = default; diff --git a/src/cache/blockcache/disk_cache.cpp b/src/cache/blockcache/disk_cache.cpp index 65dd7a6ba..e6043978f 100644 --- a/src/cache/blockcache/disk_cache.cpp +++ b/src/cache/blockcache/disk_cache.cpp @@ -76,14 +76,14 @@ DiskCache::DiskCache(DiskCacheOption option) } return status; }); - manager_ = std::make_shared(option.cache_size, layout_, fs_, - metric_); + manager_ = std::make_shared(option.cache_size(), layout_, + fs_, metric_); loader_ = std::make_unique(layout_, fs_, manager_, metric_); std::shared_ptr io_ring; if (option_.filesystem_type == "3fs") { io_ring = - std::make_shared(option_.cache_dir, option_.ioring_blksize); + std::make_shared(option_.cache_dir(), option_.ioring_blksize()); } else { io_ring = std::make_shared(); } @@ -95,7 +95,7 @@ Status DiskCache::Init(UploadFunc uploader) { return Status::OK(); // already running } - Status status = aio_queue_->Init(option_.ioring_iodepth); + Status status = aio_queue_->Init(option_.ioring_iodepth()); if (status.ok()) { status = CreateDirs(); if (status.ok()) { diff --git a/src/cache/blockcache/disk_cache_metric.h b/src/cache/blockcache/disk_cache_metric.h index c6c22bfce..c43fd7a64 100644 --- a/src/cache/blockcache/disk_cache_metric.h +++ b/src/cache/blockcache/disk_cache_metric.h @@ -59,7 +59,7 @@ class DiskCacheMetric { virtual ~DiskCacheMetric() = default; void Init() { - metric_.dir.set_value(option_.cache_dir); + metric_.dir.set_value(option_.cache_dir()); metric_.used_bytes.set_value(0); metric_.capacity.set_value(option_.cache_size); metric_.free_space_ratio.set_value(FLAGS_disk_cache_free_space_ratio); diff --git a/src/cache/common/CMakeLists.txt b/src/cache/common/CMakeLists.txt index 3b9a6fa76..234b7e41c 100644 --- a/src/cache/common/CMakeLists.txt +++ b/src/cache/common/CMakeLists.txt @@ -36,6 +36,7 @@ target_link_libraries(cache_common dingofs_base_lib dingofs_utils dingofs_common + dingofs_options client_common ) diff --git a/src/cache/common/common.h b/src/cache/common/common.h index 1543e2324..0bd2888c8 100644 --- a/src/cache/common/common.h +++ b/src/cache/common/common.h @@ -23,16 +23,16 @@ #ifndef DINGOFS_SRC_CACHE_COMMON_COMMON_H_ #define DINGOFS_SRC_CACHE_COMMON_COMMON_H_ -#include "client/common/common.h" -#include "client/common/config.h" +#include "client/common/status.h" +#include "options/cache/app.h" namespace dingofs { namespace cache { namespace common { using Status = client::Status; -using BlockCacheOption = client::common::BlockCacheOption; -using DiskCacheOption = client::common::DiskCacheOption; +using BlockCacheOption = options::cache::BlockCacheOption; +using DiskCacheOption = options::cache::DiskCacheOption; } // namespace common } // namespace cache diff --git a/src/client/common/CMakeLists.txt b/src/client/common/CMakeLists.txt index 57bab1a81..7f31259bc 100644 --- a/src/client/common/CMakeLists.txt +++ b/src/client/common/CMakeLists.txt @@ -17,7 +17,6 @@ add_library(client_common dynamic_config.cpp slice.cpp status.cpp - config.cpp common.cpp ) diff --git a/src/client/common/config.cpp b/src/client/common/config.cpp deleted file mode 100644 index 53468cd06..000000000 --- a/src/client/common/config.cpp +++ /dev/null @@ -1,478 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Project: dingo - * Created Date: Thur May 27 2021 - * Author: xuchaojie - */ - -#include "client/common/config.h" - -#include - -#include "base/filepath/filepath.h" -#include "base/math/math.h" -#include "base/string/string.h" -#include "cache/common/dynamic_config.h" -#include "client/common/dynamic_config.h" -#include "gflags/gflags.h" -#include "utils/gflags_helper.h" - -namespace brpc { -DECLARE_int32(defer_close_second); -DECLARE_int32(health_check_interval); -} // namespace brpc - -namespace dingofs { -namespace client { -namespace common { - -using ::dingofs::base::filepath::PathJoin; -using ::dingofs::base::math::kMiB; -using ::dingofs::base::string::Str2Int; -using ::dingofs::dataaccess::aws::S3InfoOption; -using ::dingofs::utils::Configuration; - -using ::dingofs::base::string::StrSplit; -using ::dingofs::stub::common::ExcutorOpt; -using ::dingofs::stub::common::MetaCacheOpt; - -using dingofs::client::common::FLAGS_in_time_warmup; - -static void InitMetaCacheOption(Configuration* conf, MetaCacheOpt* opts) { - conf->GetValueFatalIfFail("metaCacheOpt.metacacheGetLeaderRetry", - &opts->metacacheGetLeaderRetry); - conf->GetValueFatalIfFail("metaCacheOpt.metacacheRPCRetryIntervalUS", - &opts->metacacheRPCRetryIntervalUS); - conf->GetValueFatalIfFail("metaCacheOpt.metacacheGetLeaderRPCTimeOutMS", - &opts->metacacheGetLeaderRPCTimeOutMS); -} - -static void InitExcutorOption(Configuration* conf, ExcutorOpt* opts, - bool internal) { - if (internal) { - conf->GetValueFatalIfFail("executorOpt.maxInternalRetry", &opts->maxRetry); - } else { - conf->GetValueFatalIfFail("executorOpt.maxRetry", &opts->maxRetry); - } - - conf->GetValueFatalIfFail("executorOpt.retryIntervalUS", - &opts->retryIntervalUS); - conf->GetValueFatalIfFail("executorOpt.rpcTimeoutMS", &opts->rpcTimeoutMS); - conf->GetValueFatalIfFail("executorOpt.rpcStreamIdleTimeoutMS", - &opts->rpcStreamIdleTimeoutMS); - conf->GetValueFatalIfFail("executorOpt.maxRPCTimeoutMS", - &opts->maxRPCTimeoutMS); - conf->GetValueFatalIfFail("executorOpt.maxRetrySleepIntervalUS", - &opts->maxRetrySleepIntervalUS); - conf->GetValueFatalIfFail("executorOpt.minRetryTimesForceTimeoutBackoff", - &opts->minRetryTimesForceTimeoutBackoff); - conf->GetValueFatalIfFail("executorOpt.maxRetryTimesBeforeConsiderSuspend", - &opts->maxRetryTimesBeforeConsiderSuspend); - conf->GetValueFatalIfFail("executorOpt.batchInodeAttrLimit", - &opts->batchInodeAttrLimit); - conf->GetValueFatalIfFail("fuseClient.enableMultiMountPointRename", - &opts->enableRenameParallel); -} - -static void InitS3Option(Configuration* conf, S3Option* s3_opt) { - conf->GetValueFatalIfFail("s3.fakeS3", &FLAGS_useFakeS3); - conf->GetValueFatalIfFail("data_stream.page.size", - &s3_opt->s3ClientAdaptorOpt.pageSize); - conf->GetValueFatalIfFail("s3.prefetchBlocks", - &s3_opt->s3ClientAdaptorOpt.prefetchBlocks); - conf->GetValueFatalIfFail("s3.prefetchExecQueueNum", - &s3_opt->s3ClientAdaptorOpt.prefetchExecQueueNum); - conf->GetValueFatalIfFail("data_stream.background_flush.interval_ms", - &s3_opt->s3ClientAdaptorOpt.intervalMs); - conf->GetValueFatalIfFail("data_stream.slice.stay_in_memory_max_second", - &s3_opt->s3ClientAdaptorOpt.flushIntervalSec); - conf->GetValueFatalIfFail("s3.writeCacheMaxByte", - &s3_opt->s3ClientAdaptorOpt.writeCacheMaxByte); - conf->GetValueFatalIfFail("s3.readCacheMaxByte", - &s3_opt->s3ClientAdaptorOpt.readCacheMaxByte); - conf->GetValueFatalIfFail("s3.readCacheThreads", - &s3_opt->s3ClientAdaptorOpt.readCacheThreads); - conf->GetValueFatalIfFail("s3.nearfullRatio", - &s3_opt->s3ClientAdaptorOpt.nearfullRatio); - conf->GetValueFatalIfFail("s3.baseSleepUs", - &s3_opt->s3ClientAdaptorOpt.baseSleepUs); - conf->GetValueFatalIfFail("s3.maxReadRetryIntervalMs", - &s3_opt->s3ClientAdaptorOpt.maxReadRetryIntervalMs); - conf->GetValueFatalIfFail("s3.readRetryIntervalMs", - &s3_opt->s3ClientAdaptorOpt.readRetryIntervalMs); - dataaccess::aws::InitS3AdaptorOptionExceptS3InfoOption(conf, - &s3_opt->s3AdaptrOpt); -} - -static void InitLeaseOpt(Configuration* conf, LeaseOpt* lease_opt) { - conf->GetValueFatalIfFail("mds.leaseTimesUs", &lease_opt->leaseTimeUs); - conf->GetValueFatalIfFail("mds.refreshTimesPerLease", - &lease_opt->refreshTimesPerLease); -} - -void InitUdsOption(Configuration* conf, UdsOption* uds_opt) { - if (conf->GetValue("uds.fdCommPath", &uds_opt->fd_comm_path)) { - uds_opt->fd_comm_path = "/var/run"; - } -} - -static void InitRefreshDataOpt(Configuration* conf, RefreshDataOption* opt) { - conf->GetValueFatalIfFail("fuseClient.maxDataSize", &opt->maxDataSize); - conf->GetValueFatalIfFail("fuseClient.refreshDataIntervalSec", - &opt->refreshDataIntervalSec); -} - -static void InitKVClientManagerOpt(Configuration* conf, - KVClientManagerOpt* config) { - conf->GetValueFatalIfFail("fuseClient.supportKVcache", &FLAGS_supportKVcache); - conf->GetValueFatalIfFail("fuseClient.setThreadPool", - &config->setThreadPooln); - conf->GetValueFatalIfFail("fuseClient.getThreadPool", - &config->getThreadPooln); -} - -static void InitFileSystemOption(Configuration* c, FileSystemOption* option) { - c->GetValueFatalIfFail("fs.cto", &option->cto); - c->GetValueFatalIfFail("fs.cto", &FLAGS_enableCto); - c->GetValueFatalIfFail("fs.nocto_suffix", &option->nocto_suffix); - c->GetValueFatalIfFail("fs.disableXAttr", &option->disableXAttr); - c->GetValueFatalIfFail("fs.maxNameLength", &option->maxNameLength); - c->GetValueFatalIfFail("fs.accessLogging", &FLAGS_access_logging); - { // kernel cache option - auto* o = &option->kernelCacheOption; - c->GetValueFatalIfFail("fs.kernelCache.attrTimeoutSec", &o->attrTimeoutSec); - c->GetValueFatalIfFail("fs.kernelCache.dirAttrTimeoutSec", - &o->dirAttrTimeoutSec); - c->GetValueFatalIfFail("fs.kernelCache.entryTimeoutSec", - &o->entryTimeoutSec); - c->GetValueFatalIfFail("fs.kernelCache.dirEntryTimeoutSec", - &o->dirEntryTimeoutSec); - } - { // lookup cache option - auto* o = &option->lookupCacheOption; - c->GetValueFatalIfFail("fs.lookupCache.lruSize", &o->lruSize); - c->GetValueFatalIfFail("fs.lookupCache.negativeTimeoutSec", - &o->negativeTimeoutSec); - c->GetValueFatalIfFail("fs.lookupCache.minUses", &o->minUses); - } - { // dir cache option - auto* o = &option->dirCacheOption; - c->GetValueFatalIfFail("fs.dirCache.lruSize", &o->lruSize); - } - { // attr watcher option - auto* o = &option->attrWatcherOption; - c->GetValueFatalIfFail("fs.attrWatcher.lruSize", &o->lruSize); - } - { // rpc option - auto* o = &option->rpcOption; - c->GetValueFatalIfFail("fs.rpc.listDentryLimit", &o->listDentryLimit); - } - { // defer sync option - auto* o = &option->deferSyncOption; - c->GetValueFatalIfFail("fs.deferSync.delay", &o->delay); - c->GetValueFatalIfFail("fs.deferSync.deferDirMtime", &o->deferDirMtime); - } -} - -static void InitDataStreamOption(Configuration* c, DataStreamOption* option) { - { // background flush option - auto* o = &option->background_flush_option; - c->GetValueFatalIfFail( - "data_stream.background_flush.trigger_force_memory_ratio", - &o->trigger_force_memory_ratio); - } - { // file option - auto* o = &option->file_option; - c->GetValueFatalIfFail("data_stream.file.flush_workers", &o->flush_workers); - c->GetValueFatalIfFail("data_stream.file.flush_queue_size", - &o->flush_queue_size); - } - { // chunk option - auto* o = &option->chunk_option; - c->GetValueFatalIfFail("data_stream.chunk.flush_workers", - &o->flush_workers); - c->GetValueFatalIfFail("data_stream.chunk.flush_queue_size", - &o->flush_queue_size); - } - { // slice option - auto* o = &option->slice_option; - c->GetValueFatalIfFail("data_stream.slice.flush_workers", - &o->flush_workers); - c->GetValueFatalIfFail("data_stream.slice.flush_queue_size", - &o->flush_queue_size); - } - { // page option - auto* o = &option->page_option; - c->GetValueFatalIfFail("data_stream.page.size", &o->page_size); - c->GetValueFatalIfFail("data_stream.page.total_size_mb", &o->total_size); - c->GetValueFatalIfFail("data_stream.page.use_pool", &o->use_pool); - - if (o->page_size == 0) { - CHECK(false) << "Page size must greater than 0."; - } - - o->total_size = o->total_size * kMiB; - if (o->total_size < 64 * kMiB) { - CHECK(false) << "Page total size must greater than 64MiB."; - } - - double trigger_force_flush_memory_ratio = - option->background_flush_option.trigger_force_memory_ratio; - if (o->total_size * (1.0 - trigger_force_flush_memory_ratio) < 32 * kMiB) { - CHECK(false) << "Please gurantee the free memory size greater than 32MiB " - "before force flush."; - } - } -} - -static void InitFuseOption(Configuration* c, FuseOption* option) { - { // fuse conn info - auto* o = &option->conn_info; - c->GetValueFatalIfFail("fuse.conn_info.want_splice_move", - &o->want_splice_move); - c->GetValueFatalIfFail("fuse.conn_info.want_splice_read", - &o->want_splice_read); - c->GetValueFatalIfFail("fuse.conn_info.want_splice_write", - &o->want_splice_write); - c->GetValueFatalIfFail("fuse.conn_info.want_auto_inval_data", - &o->want_auto_inval_data); - } - - { // fuse file info - c->GetValueFatalIfFail("fuse.file_info.direct_io", - &FLAGS_fuse_file_info_direct_io); - c->GetValueFatalIfFail("fuse.file_info.keep_cache", - &FLAGS_fuse_file_info_keep_cache); - } -} - -namespace { - -void SplitDiskCacheOption(DiskCacheOption option, - std::vector* options) { - std::vector dirs = StrSplit(option.cache_dir, ";"); - for (size_t i = 0; i < dirs.size(); i++) { - uint64_t cache_size = option.cache_size; - std::vector items = StrSplit(dirs[i], ":"); - if (items.size() > 2 || - (items.size() == 2 && !Str2Int(items[1], &cache_size))) { - CHECK(false) << "Invalid cache dir: " << dirs[i]; - } else if (cache_size == 0) { - CHECK(false) << "Cache size must greater than 0."; - } - - DiskCacheOption o = option; - o.index = i; - o.cache_dir = items[0]; - o.cache_size = cache_size * kMiB; - options->emplace_back(o); - } -} - -} // namespace - -static void InitBlockCacheOption(Configuration* c, BlockCacheOption* option) { - { // block cache option - USING_CACHE_FLAG(block_cache_stage_bandwidth_throttle_enable); - USING_CACHE_FLAG(block_cache_stage_bandwidth_throttle_mb); - USING_CACHE_FLAG(trace_logging); - - c->GetValueFatalIfFail("block_cache.stage", &option->stage); - c->GetValueFatalIfFail("block_cache.stage_bandwidth_throttle_enable", - &FLAGS_block_cache_stage_bandwidth_throttle_enable); - c->GetValueFatalIfFail("block_cache.stage_bandwidth_throttle_mb", - &FLAGS_block_cache_stage_bandwidth_throttle_mb); - c->GetValueFatalIfFail("block_cache.logging", &FLAGS_trace_logging); - c->GetValueFatalIfFail("block_cache.upload_stage_workers", - &option->upload_stage_workers); - c->GetValueFatalIfFail("block_cache.upload_stage_queue_size", - &option->upload_stage_queue_size); - c->GetValueFatalIfFail("block_cache.prefetch_workers", - &option->prefetch_workers); - c->GetValueFatalIfFail("block_cache.prefetch_queue_size", - &option->prefetch_queue_size); - c->GetValueFatalIfFail("block_cache.cache_store", &option->cache_store); - - std::string cache_store = option->cache_store; - if (cache_store != "none" && cache_store != "disk" && - cache_store != "3fs") { - CHECK(false) << "Only support none, disk or 3fs cache store."; - } - } - - { // disk cache option - USING_CACHE_FLAG(disk_cache_free_space_ratio); - USING_CACHE_FLAG(disk_cache_expire_second); - USING_CACHE_FLAG(disk_cache_cleanup_expire_interval_millsecond); - USING_CACHE_FLAG(drop_page_cache); - - DiskCacheOption o; - c->GetValueFatalIfFail("disk_cache.cache_dir", &o.cache_dir); - c->GetValueFatalIfFail("disk_cache.cache_size_mb", &o.cache_size); - c->GetValueFatalIfFail("disk_cache.free_space_ratio", - &FLAGS_disk_cache_free_space_ratio); - c->GetValueFatalIfFail("disk_cache.cache_expire_second", - &FLAGS_disk_cache_expire_second); - c->GetValueFatalIfFail( - "disk_cache.cleanup_expire_interval_millsecond", - &FLAGS_disk_cache_cleanup_expire_interval_millsecond); - c->GetValueFatalIfFail("disk_cache.drop_page_cache", - &FLAGS_drop_page_cache); - if (!c->GetUInt32Value("disk_cache.ioring_iodepth", &o.ioring_iodepth)) { - o.ioring_iodepth = 128; - } - if (!c->GetUInt32Value("disk_cache.ioring_blksize", &o.ioring_blksize)) { - o.ioring_blksize = 1048576; - } - o.ioring_prefetch = c->GetBoolValue("disk_cache.ioring_prefetch", true); - - if (option->cache_store == "disk" || option->cache_store == "3fs") { - o.filesystem_type = (option->cache_store == "3fs") ? "3fs" : "local"; - SplitDiskCacheOption(o, &option->disk_cache_options); - } - } - - { // disk state option - USING_CACHE_FLAG(disk_state_tick_duration_second); - USING_CACHE_FLAG(disk_state_normal2unstable_io_error_num); - USING_CACHE_FLAG(disk_state_unstable2normal_io_succ_num); - USING_CACHE_FLAG(disk_state_unstable2down_second); - USING_CACHE_FLAG(disk_check_duration_millsecond); - - c->GetValueFatalIfFail("disk_state.tick_duration_second", - &FLAGS_disk_state_tick_duration_second); - c->GetValueFatalIfFail("disk_state.normal2unstable_io_error_num", - &FLAGS_disk_state_normal2unstable_io_error_num); - c->GetValueFatalIfFail("disk_state.unstable2normal_io_succ_num", - &FLAGS_disk_state_unstable2normal_io_succ_num); - c->GetValueFatalIfFail("disk_state.unstable2down_second", - &FLAGS_disk_state_unstable2down_second); - c->GetValueFatalIfFail("disk_state.disk_check_duration_millsecond", - &FLAGS_disk_check_duration_millsecond); - } -} - -static void SetBrpcOpt(Configuration* conf) { - dingofs::utils::GflagsLoadValueFromConfIfCmdNotSet dummy; - dummy.Load(conf, "defer_close_second", "rpc.defer.close.second", - &brpc::FLAGS_defer_close_second); - dummy.Load(conf, "health_check_interval", "rpc.healthCheckIntervalSec", - &brpc::FLAGS_health_check_interval); -} - -void InitClientOption(Configuration* conf, ClientOption* client_option) { - InitMdsOption(conf, &client_option->mdsOpt); - InitMetaCacheOption(conf, &client_option->metaCacheOpt); - InitExcutorOption(conf, &client_option->excutorOpt, false); - InitExcutorOption(conf, &client_option->excutorInternalOpt, true); - InitS3Option(conf, &client_option->s3Opt); - InitLeaseOpt(conf, &client_option->leaseOpt); - InitRefreshDataOpt(conf, &client_option->refreshDataOption); - InitKVClientManagerOpt(conf, &client_option->kvClientManagerOpt); - InitFileSystemOption(conf, &client_option->fileSystemOption); - InitDataStreamOption(conf, &client_option->data_stream_option); - InitBlockCacheOption(conf, &client_option->block_cache_option); - InitFuseOption(conf, &client_option->fuse_option); - - conf->GetValueFatalIfFail("fuseClient.listDentryLimit", - &client_option->listDentryLimit); - conf->GetValueFatalIfFail("fuseClient.listDentryThreads", - &client_option->listDentryThreads); - conf->GetValueFatalIfFail("client.dummyServer.startPort", - &client_option->dummyServerStartPort); - conf->GetValueFatalIfFail("fuseClient.enableMultiMountPointRename", - &client_option->enableMultiMountPointRename); - conf->GetValueFatalIfFail("fuseClient.downloadMaxRetryTimes", - &client_option->downloadMaxRetryTimes); - conf->GetValueFatalIfFail("fuseClient.warmupThreadsNum", - &client_option->warmupThreadsNum); - if (!conf->GetBoolValue("fuseClient.in_time_warmup", &FLAGS_in_time_warmup)) { - LOG(INFO) << "Not found `fuseClient.in_time_warmup` in conf, default to " - "false"; - FLAGS_in_time_warmup = false; - } - - if (!conf->GetIntValue("fuseClient.bthread_worker_num", - &FLAGS_bthread_worker_num)) { - FLAGS_bthread_worker_num = 0; - LOG(INFO) - << "Not found `fuseClient.bthread_worker_num` in conf, default to 0"; - } - - conf->GetValueFatalIfFail("fuseClient.throttle.avgWriteBytes", - &FLAGS_fuseClientAvgWriteBytes); - conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteBytes", - &FLAGS_fuseClientBurstWriteBytes); - conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteBytesSecs", - &FLAGS_fuseClientBurstWriteBytesSecs); - - conf->GetValueFatalIfFail("fuseClient.throttle.avgWriteIops", - &FLAGS_fuseClientAvgWriteIops); - conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteIops", - &FLAGS_fuseClientBurstWriteIops); - conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteIopsSecs", - &FLAGS_fuseClientBurstWriteIopsSecs); - - conf->GetValueFatalIfFail("fuseClient.throttle.avgReadBytes", - &FLAGS_fuseClientAvgReadBytes); - conf->GetValueFatalIfFail("fuseClient.throttle.burstReadBytes", - &FLAGS_fuseClientBurstReadBytes); - conf->GetValueFatalIfFail("fuseClient.throttle.burstReadBytesSecs", - &FLAGS_fuseClientBurstReadBytesSecs); - - conf->GetValueFatalIfFail("fuseClient.throttle.avgReadIops", - &FLAGS_fuseClientAvgReadIops); - conf->GetValueFatalIfFail("fuseClient.throttle.burstReadIops", - &FLAGS_fuseClientBurstReadIops); - conf->GetValueFatalIfFail("fuseClient.throttle.burstReadIopsSecs", - &FLAGS_fuseClientBurstReadIopsSecs); - SetBrpcOpt(conf); -} - -void SetClientS3Option(ClientOption* client_option, - const S3InfoOption& fs_s3_opt) { - client_option->s3Opt.s3ClientAdaptorOpt.blockSize = fs_s3_opt.blockSize; - client_option->s3Opt.s3ClientAdaptorOpt.chunkSize = fs_s3_opt.chunkSize; - client_option->s3Opt.s3ClientAdaptorOpt.objectPrefix = fs_s3_opt.objectPrefix; - client_option->s3Opt.s3AdaptrOpt.s3Address = fs_s3_opt.s3Address; - client_option->s3Opt.s3AdaptrOpt.ak = fs_s3_opt.ak; - client_option->s3Opt.s3AdaptrOpt.sk = fs_s3_opt.sk; - client_option->s3Opt.s3AdaptrOpt.bucketName = fs_s3_opt.bucketName; -} - -void S3Info2FsS3Option(const pb::common::S3Info& s3, S3InfoOption* fs_s3_opt) { - fs_s3_opt->ak = s3.ak(); - fs_s3_opt->sk = s3.sk(); - fs_s3_opt->s3Address = s3.endpoint(); - fs_s3_opt->bucketName = s3.bucketname(); - fs_s3_opt->blockSize = s3.blocksize(); - fs_s3_opt->chunkSize = s3.chunksize(); - fs_s3_opt->objectPrefix = s3.has_objectprefix() ? s3.objectprefix() : 0; -} - -void RewriteCacheDir(BlockCacheOption* option, std::string uuid) { - auto& disk_cache_options = option->disk_cache_options; - for (auto& disk_cache_option : disk_cache_options) { - std::string cache_dir = disk_cache_option.cache_dir; - disk_cache_option.cache_dir = PathJoin({cache_dir, uuid}); - } -} - -} // namespace common -} // namespace client -} // namespace dingofs diff --git a/src/client/common/config.h b/src/client/common/config.h deleted file mode 100644 index d3152b92b..000000000 --- a/src/client/common/config.h +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Project: dingo - * Created Date: Thur May 27 2021 - * Author: xuchaojie - */ - -#ifndef DINGOFS_SRC_CLIENT_COMMON_CONFIG_H_ -#define DINGOFS_SRC_CLIENT_COMMON_CONFIG_H_ - -#include -#include - -#include "dataaccess/aws/s3_adapter.h" -#include "dingofs/common.pb.h" -#include "stub/common/config.h" -#include "utils/configuration.h" - -namespace dingofs { -namespace client { -namespace common { - -// { data stream option -struct BackgroundFlushOption { - double trigger_force_memory_ratio; -}; - -struct FileOption { - uint64_t flush_workers; - uint64_t flush_queue_size; -}; - -struct SliceOption { - uint64_t flush_workers; - uint64_t flush_queue_size; -}; - -struct ChunkOption { - uint64_t flush_workers; - uint64_t flush_queue_size; -}; - -struct PageOption { - uint64_t page_size; - uint64_t total_size; - bool use_pool; -}; - -struct DataStreamOption { - BackgroundFlushOption background_flush_option; - FileOption file_option; - ChunkOption chunk_option; - SliceOption slice_option; - PageOption page_option; -}; -// } - -// { block cache option -struct DiskCacheOption { - uint32_t index; - std::string cache_dir; - uint64_t cache_size; // bytes - std::string filesystem_type; // local or 3fs - uint32_t ioring_iodepth; - uint32_t ioring_blksize; - bool ioring_prefetch; -}; - -struct BlockCacheOption { - std::string cache_store; - bool stage; - bool stage_throttle_enable; - uint64_t stage_throttle_bandwidth_mb; - uint32_t flush_file_workers; - uint32_t flush_file_queue_size; - uint32_t flush_slice_workers; - uint32_t flush_slice_queue_size; - uint32_t upload_stage_workers; - uint32_t upload_stage_queue_size; - uint32_t prefetch_workers; - uint32_t prefetch_queue_size; - std::vector disk_cache_options; -}; -// } - -struct LeaseOpt { - uint32_t refreshTimesPerLease = 5; - // default = 20s - uint32_t leaseTimeUs = 20000000; -}; - -struct KVClientManagerOpt { - int setThreadPooln = 4; - int getThreadPooln = 4; -}; - -struct S3ClientAdaptorOption { - uint64_t blockSize; - uint64_t chunkSize; - uint64_t pageSize; - uint32_t prefetchBlocks; - uint32_t prefetchExecQueueNum; - uint32_t intervalMs; - uint32_t flushIntervalSec; - uint64_t writeCacheMaxByte; - uint64_t readCacheMaxByte; - uint32_t readCacheThreads; - uint32_t nearfullRatio; - uint32_t baseSleepUs; - uint32_t maxReadRetryIntervalMs; - uint32_t readRetryIntervalMs; - uint32_t objectPrefix; -}; - -struct S3Option { - S3ClientAdaptorOption s3ClientAdaptorOpt; - dataaccess::aws::S3AdapterOption s3AdaptrOpt; -}; - -struct RefreshDataOption { - uint64_t maxDataSize = 1024; - uint32_t refreshDataIntervalSec = 30; -}; - -// { fuse module option -struct FuseConnInfo { - bool want_splice_move; - bool want_splice_read; - bool want_splice_write; - bool want_auto_inval_data; -}; - -struct FuseFileInfo { - bool keep_cache; -}; - -struct FuseOption { - FuseConnInfo conn_info; - FuseFileInfo file_info; -}; -// } - -// { filesystem option -struct KernelCacheOption { - uint32_t entryTimeoutSec; - uint32_t dirEntryTimeoutSec; - uint32_t attrTimeoutSec; - uint32_t dirAttrTimeoutSec; -}; - -struct LookupCacheOption { - uint64_t lruSize; - uint32_t negativeTimeoutSec; - uint32_t minUses; -}; - -struct DirCacheOption { - uint64_t lruSize; - uint32_t timeoutSec; -}; - -struct AttrWatcherOption { - uint64_t lruSize; -}; - -struct OpenFilesOption { - uint64_t lruSize; - uint32_t deferSyncSecond; -}; - -struct RPCOption { - uint32_t listDentryLimit; -}; - -struct DeferSyncOption { - uint32_t delay; - bool deferDirMtime; -}; - -struct FileSystemOption { - bool cto; - std::string nocto_suffix; - bool disableXAttr; - uint32_t maxNameLength; - uint32_t blockSize = 0x10000u; - KernelCacheOption kernelCacheOption; - LookupCacheOption lookupCacheOption; - DirCacheOption dirCacheOption; - OpenFilesOption openFilesOption; - AttrWatcherOption attrWatcherOption; - RPCOption rpcOption; - DeferSyncOption deferSyncOption; -}; - -struct UdsOption { - std::string fd_comm_path; -}; -// } - -struct ClientOption { - stub::common::MdsOption mdsOpt; - stub::common::MetaCacheOpt metaCacheOpt; - stub::common::ExcutorOpt excutorOpt; - stub::common::ExcutorOpt excutorInternalOpt; - S3Option s3Opt; - LeaseOpt leaseOpt; - RefreshDataOption refreshDataOption; - KVClientManagerOpt kvClientManagerOpt; - FileSystemOption fileSystemOption; - DataStreamOption data_stream_option; - BlockCacheOption block_cache_option; - FuseOption fuse_option; - - uint32_t listDentryLimit; - uint32_t listDentryThreads; - uint32_t dummyServerStartPort; - bool enableMultiMountPointRename = false; - uint32_t downloadMaxRetryTimes; - uint32_t warmupThreadsNum = 10; -}; - -void InitClientOption(utils::Configuration* conf, ClientOption* client_option); - -void SetClientS3Option(ClientOption* client_option, - const dataaccess::aws::S3InfoOption& fs_s3_opt); - -void S3Info2FsS3Option(const pb::common::S3Info& s3, - dataaccess::aws::S3InfoOption* fs_s3_opt); - -void RewriteCacheDir(BlockCacheOption* option, std::string uuid); - -void InitUdsOption(utils::Configuration* conf, UdsOption* uds_option); - -} // namespace common -} // namespace client -} // namespace dingofs - -#endif // DINGOFS_SRC_CLIENT_COMMON_CONFIG_H_ diff --git a/src/client/datastream/data_stream.cpp b/src/client/datastream/data_stream.cpp index fbfdcf82e..68e425720 100644 --- a/src/client/datastream/data_stream.cpp +++ b/src/client/datastream/data_stream.cpp @@ -27,7 +27,7 @@ #include #include -#include "client/common/config.h" +#include "base/math/math.h" #include "client/common/share_var.h" #include "client/datastream/metric.h" #include "client/datastream/page_allocator.h" @@ -36,18 +36,19 @@ namespace dingofs { namespace client { namespace datastream { +using base::math::kMiB; using ::dingofs::client::common::ShareVar; -bool DataStream::Init(DataStreamOption option) { +bool DataStream::Init(const DataOption& option) { option_ = option; // file { - auto o = option.file_option; + auto o = option.flush_file_option(); flush_file_thread_pool_ = std::make_shared>("flush_file_worker"); auto rc = - flush_file_thread_pool_->Start(o.flush_workers, o.flush_queue_size); + flush_file_thread_pool_->Start(o.flush_workers(), o.flush_queue_size()); if (rc != 0) { LOG(ERROR) << "Start flush file thread pool failed, rc = " << rc; return false; @@ -56,11 +57,11 @@ bool DataStream::Init(DataStreamOption option) { // chunk { - auto o = option.chunk_option; + auto o = option.flush_chunk_option(); flush_chunk_thread_pool_ = std::make_shared>("flush_chunk_worker"); - auto rc = - flush_chunk_thread_pool_->Start(o.flush_workers, o.flush_queue_size); + auto rc = flush_chunk_thread_pool_->Start(o.flush_workers(), + o.flush_queue_size()); if (rc != 0) { LOG(ERROR) << "Start flush chunk thread pool failed, rc = " << rc; return false; @@ -69,11 +70,11 @@ bool DataStream::Init(DataStreamOption option) { // slice { - auto o = option.slice_option; + auto o = option.flush_slice_option(); flush_slice_thread_pool_ = std::make_shared>("flush_slice_worker"); - auto rc = - flush_slice_thread_pool_->Start(o.flush_workers, o.flush_queue_size); + auto rc = flush_slice_thread_pool_->Start(o.flush_workers(), + o.flush_queue_size()); if (rc != 0) { LOG(ERROR) << "Start flush slice thread pool failed, rc = " << rc; return false; @@ -82,19 +83,21 @@ bool DataStream::Init(DataStreamOption option) { // page { - auto o = option.page_option; + auto o = option.page_option(); // Smooth upgrade use DefaultPageAllocator + bool use_pool = o.use_pool(); if (ShareVar::GetInstance().HasValue(common::kSmoothUpgradeNew)) { LOG(INFO) << "use default page allocate for smooth upgrade"; - o.use_pool = false; + use_pool = false; } - if (o.use_pool) { + if (use_pool) { page_allocator_ = std::make_shared(); } else { page_allocator_ = std::make_shared(); } - auto ok = page_allocator_->Init(o.page_size, o.total_size / o.page_size); + auto ok = page_allocator_->Init( + o.page_size(), o.total_size_mb() * base::math::kMiB / o.page_size()); if (!ok) { LOG(ERROR) << "Init page allocator failed."; return false; @@ -136,9 +139,9 @@ void DataStream::FreePage(char* page) { page_allocator_->DeAllocate(page); } bool DataStream::MemoryNearFull() { double trigger_force_memory_ratio = - option_.background_flush_option.trigger_force_memory_ratio; - uint64_t page_size = option_.page_option.page_size; - uint64_t total_size = option_.page_option.total_size; + option_.background_flush_option().trigger_force_memory_ratio(); + uint64_t page_size = option_.page_option().page_size(); + uint64_t total_size = option_.page_option().total_size_mb() * kMiB; bool is_full = page_allocator_->GetFreePages() * page_size <= total_size * (1.0 - trigger_force_memory_ratio); return is_full; diff --git a/src/client/datastream/data_stream.h b/src/client/datastream/data_stream.h index f4fdce571..ec363e333 100644 --- a/src/client/datastream/data_stream.h +++ b/src/client/datastream/data_stream.h @@ -27,9 +27,9 @@ #include #include -#include "client/common/config.h" #include "client/datastream/metric.h" #include "client/datastream/page_allocator.h" +#include "options/client/data.h" #include "utils/concurrent/task_thread_pool.h" namespace dingofs { @@ -37,7 +37,7 @@ namespace client { namespace datastream { using ::dingofs::utils::TaskThreadPool; -using ::dingofs::client::common::DataStreamOption; +using options::client::DataOption; class DataStream { using TaskFunc = std::function; @@ -49,7 +49,7 @@ class DataStream { } public: - bool Init(DataStreamOption option); + bool Init(const DataOption& option); void Shutdown(); @@ -71,7 +71,7 @@ class DataStream { std::shared_ptr> flush_slice_thread_pool_; std::shared_ptr page_allocator_; std::unique_ptr metric_; - DataStreamOption option_; + DataOption option_; }; } // namespace datastream diff --git a/src/client/datastream/metric.h b/src/client/datastream/metric.h index 664e51a4a..9d96c41b8 100644 --- a/src/client/datastream/metric.h +++ b/src/client/datastream/metric.h @@ -28,8 +28,8 @@ #include -#include "client/common/config.h" #include "client/datastream/page_allocator.h" +#include "options/client/data.h" #include "utils/concurrent/task_thread_pool.h" namespace dingofs { @@ -37,7 +37,7 @@ namespace client { namespace datastream { using ::dingofs::utils::TaskThreadPool; -using ::dingofs::client::common::DataStreamOption; +using options::client::DataOption; static uint32_t GetQueueSize(void* arg) { auto* thread_pool = reinterpret_cast*>(arg); @@ -59,33 +59,33 @@ class DataStreamMetric { }; public: - DataStreamMetric(DataStreamOption option, AuxMembers aux_members) + DataStreamMetric(DataOption option, AuxMembers aux_members) : metric_("dingofs_data_stream", aux_members) { // file { - auto o = option.file_option; - metric_.flush_file_workers.set_value(o.flush_workers); - metric_.flush_file_queue_capacity.set_value(o.flush_queue_size); + auto o = option.flush_file_option(); + metric_.flush_file_workers.set_value(o.flush_workers()); + metric_.flush_file_queue_capacity.set_value(o.flush_queue_size()); } // chunk { - auto o = option.chunk_option; - metric_.flush_chunk_workers.set_value(o.flush_workers); - metric_.flush_chunk_queue_capacity.set_value(o.flush_queue_size); + auto o = option.flush_chunk_option(); + metric_.flush_chunk_workers.set_value(o.flush_workers()); + metric_.flush_chunk_queue_capacity.set_value(o.flush_queue_size()); } // slice { - auto o = option.slice_option; - metric_.flush_slice_workers.set_value(o.flush_workers); - metric_.flush_slice_queue_capacity.set_value(o.flush_queue_size); + auto o = option.flush_slice_option(); + metric_.flush_slice_workers.set_value(o.flush_workers()); + metric_.flush_slice_queue_capacity.set_value(o.flush_queue_size()); } // page { - auto o = option.page_option; - metric_.use_page_pool.set_value(o.use_pool); + auto o = option.page_option(); + metric_.use_page_pool.set_value(o.use_pool()); } } diff --git a/src/client/main.cpp b/src/client/main.cpp index d1f654b64..3749bcc43 100644 --- a/src/client/main.cpp +++ b/src/client/main.cpp @@ -21,9 +21,11 @@ #include "client/fuse/fuse_op.h" #include "client/fuse/fuse_server.h" #include "client/vfs_wrapper/global_log.h" -#include "common/config.h" +#include "options/client/app.h" #include "utils/configuration.h" +using dingofs::options::client::OPTIONS_client; + using FuseServer = dingofs::client::fuse::FuseServer; using Configuration = dingofs::utils::Configuration; diff --git a/src/client/vfs/meta/v2/filesystem.cpp b/src/client/vfs/meta/v2/filesystem.cpp index 2a4cfed4a..82e88969c 100644 --- a/src/client/vfs/meta/v2/filesystem.cpp +++ b/src/client/vfs/meta/v2/filesystem.cpp @@ -506,10 +506,10 @@ Status MDSV2FileSystem::Rename(Ino old_parent, const std::string& old_name, } MDSV2FileSystemUPtr MDSV2FileSystem::Build(const std::string& fs_name, - const std::string& mds_addr, - const std::string& mountpoint) { + const std::string& mountpoint, + const MetaOption& option) { LOG(INFO) << fmt::format("fs_name: {}, mds_addr: {}, mountpoint: {}.", - fs_name, mds_addr, mountpoint); + fs_name, mountpoint); CHECK(!fs_name.empty()) << "fs_name is empty."; CHECK(!mds_addr.empty()) << "mds_addr is empty."; diff --git a/src/client/vfs/meta/v2/filesystem.h b/src/client/vfs/meta/v2/filesystem.h index 701ada80b..bdeb77e07 100644 --- a/src/client/vfs/meta/v2/filesystem.h +++ b/src/client/vfs/meta/v2/filesystem.h @@ -27,6 +27,7 @@ #include "client/vfs/meta/v2/mds_discovery.h" #include "client/vfs/vfs_meta.h" #include "dingofs/mdsv2.pb.h" +#include "options/client/meta.h" namespace dingofs { namespace client { @@ -36,6 +37,7 @@ namespace v2 { class MDSV2FileSystem; using MDSV2FileSystemPtr = std::shared_ptr; using MDSV2FileSystemUPtr = std::unique_ptr; +using options::client::MetaOption; class MdsV2DirIterator : public DirIterator { public: @@ -88,8 +90,8 @@ class MDSV2FileSystem : public vfs::MetaSystem { } static MDSV2FileSystemUPtr Build(const std::string& fs_name, - const std::string& mds_addr, - const std::string& mountpoint); + const std::string& mountpoint, + const MetaOption& option); Status Init() override; diff --git a/src/client/vfs/vfs.h b/src/client/vfs/vfs.h index 24be0fecf..8cf535ee4 100644 --- a/src/client/vfs/vfs.h +++ b/src/client/vfs/vfs.h @@ -21,7 +21,6 @@ #include #include -#include "client/common/config.h" #include "client/common/status.h" #include "client/vfs/vfs_meta.h" @@ -127,11 +126,6 @@ class VFS { virtual double GetEntryTimeout(const FileType& type) = 0; virtual uint64_t GetMaxNameLength() = 0; - - // TODO: refactor this interface - // used for fuse - virtual common::FuseOption GetFuseOption() = 0; - }; } // namespace vfs diff --git a/src/client/vfs/vfs_impl.cpp b/src/client/vfs/vfs_impl.cpp index 25813e685..8388bc48c 100644 --- a/src/client/vfs/vfs_impl.cpp +++ b/src/client/vfs/vfs_impl.cpp @@ -44,25 +44,14 @@ Status VFSImpl::Start(const VFSConfig& vfs_conf) { // NOLINT MetaLogGuard log_guard( [&]() { return absl::StrFormat("init %s", s.ToString()); }); - utils::Configuration conf; - conf.SetConfigPath(vfs_conf.config_path); - if (!conf.LoadConfig()) { - LOG(ERROR) << "load config fail, confPath = " << vfs_conf.config_path; - return Status::Internal("load config fail"); - } - if (vfs_conf.fs_type == "vfs_dummy") { LOG(INFO) << "use dummy file system."; meta_system_ = std::make_unique(); } else if (vfs_conf.fs_type == "vfs_v2") { LOG(INFO) << "use mdsv2 file system."; - std::string mds_addr; - conf.GetValueFatalIfFail("mds.addr", &mds_addr); - LOG(INFO) << fmt::format("mds addr: {}.", mds_addr); - - meta_system_ = v2::MDSV2FileSystem::Build(vfs_conf.fs_name, mds_addr, - vfs_conf.mount_point); + meta_system_ = + v2::MDSV2FileSystem::Build(vfs_conf.fs_name, vfs_conf.mount_point); } else { LOG(INFO) << fmt::format("not unknown file system {}.", vfs_conf.fs_type); return Status::Internal("not unknown file system"); @@ -98,9 +87,23 @@ Status VFSImpl::Stop() { return Status::OK(); } -double VFSImpl::GetAttrTimeout(const FileType& type) { return 1; } // NOLINT +double VFSImpl::GetAttrTimeout(const FileType& type) { + const auto& option = option_.kernel_cache_option(); + if (type == FileType::kDirectory) { + return option.dir_attr_timeout_s(); + } else { + return option.attr_timeout_s(); + } +} -double VFSImpl::GetEntryTimeout(const FileType& type) { return 1; } // NOLINT +double VFSImpl::GetEntryTimeout(const FileType& type) { + const auto& option = option_.kernel_cache_option(); + if (type == FileType::kDirectory) { + return option.dir_entry_timeout_s(); + } else { + return option.entry_timeout_s(); + } +} Status VFSImpl::Lookup(Ino parent, const std::string& name, Attr* attr) { Status s; diff --git a/src/client/vfs/vfs_impl.h b/src/client/vfs/vfs_impl.h index 042e0d2f8..985644944 100644 --- a/src/client/vfs/vfs_impl.h +++ b/src/client/vfs/vfs_impl.h @@ -20,19 +20,21 @@ #include #include -#include "client/common/config.h" #include "client/vfs/handle_manager.h" #include "client/vfs/meta/meta_system.h" #include "client/vfs/vfs.h" +#include "options/client/app.h" +#include "options/client/vfs.h" namespace dingofs { namespace client { namespace vfs { +using options::client::VFSOption; + class VFSImpl : public VFS { public: - VFSImpl(const common::ClientOption& fuse_client_option) - : fuse_client_option_(fuse_client_option) {}; + VFSImpl(const VFSOption& option) : option_(option){}; ~VFSImpl() override = default; @@ -110,14 +112,10 @@ class VFSImpl : public VFS { uint64_t GetMaxNameLength() override; - common::FuseOption GetFuseOption() override { - return fuse_client_option_.fuse_option; - } - private: std::atomic_bool started_{false}; - common::ClientOption fuse_client_option_; + VFSOption option_; std::unique_ptr meta_system_; std::unique_ptr handle_manager_; diff --git a/src/client/vfs_old/CMakeLists.txt b/src/client/vfs_old/CMakeLists.txt index a07784fb6..805c11531 100644 --- a/src/client/vfs_old/CMakeLists.txt +++ b/src/client/vfs_old/CMakeLists.txt @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -add_subdirectory(kvclient) add_subdirectory(lease) file(GLOB VFS_OLD_SRC_LIB @@ -30,6 +29,7 @@ target_link_libraries(vfs_old_lib dingofs_base_lib rpcclient dingofs_common + dingofs_options client_lease client_memcached_client dynamic_vlog diff --git a/src/client/vfs_old/filesystem/attr_watcher.cpp b/src/client/vfs_old/filesystem/attr_watcher.cpp index 6dd3ac4eb..a7f48a78a 100644 --- a/src/client/vfs_old/filesystem/attr_watcher.cpp +++ b/src/client/vfs_old/filesystem/attr_watcher.cpp @@ -22,8 +22,8 @@ #include "client/vfs_old/filesystem/attr_watcher.h" -#include "dingofs/metaserver.pb.h" #include "client/vfs_old/filesystem/utils.h" +#include "dingofs/metaserver.pb.h" namespace dingofs { namespace client { @@ -36,10 +36,10 @@ using utils::WriteLockGuard; using pb::metaserver::InodeAttr; -AttrWatcher::AttrWatcher(AttrWatcherOption option, +AttrWatcher::AttrWatcher(const AttrWatcherOption& option, std::shared_ptr openFiles, std::shared_ptr dirCache) - : modifiedAt_(std::make_shared(option.lruSize)), + : modifiedAt_(std::make_shared(option.lru_size())), openFiles_(openFiles), dirCache_(dirCache) {} diff --git a/src/client/vfs_old/filesystem/attr_watcher.h b/src/client/vfs_old/filesystem/attr_watcher.h index be5b2d3ce..220512860 100644 --- a/src/client/vfs_old/filesystem/attr_watcher.h +++ b/src/client/vfs_old/filesystem/attr_watcher.h @@ -25,21 +25,23 @@ #include -#include "client/common/config.h" #include "client/vfs_old/filesystem/dir_cache.h" #include "client/vfs_old/filesystem/meta.h" #include "client/vfs_old/filesystem/openfile.h" +#include "options/client/vfs.h" #include "utils/lru_cache.h" namespace dingofs { namespace client { namespace filesystem { +using options::client::AttrWatcherOption; + class AttrWatcher { public: using LRUType = utils::LRUCache; - AttrWatcher(common::AttrWatcherOption option, + AttrWatcher(const AttrWatcherOption& option, std::shared_ptr openFiles, std::shared_ptr dirCache); diff --git a/src/client/vfs_old/filesystem/defer_sync.cpp b/src/client/vfs_old/filesystem/defer_sync.cpp index 44c21f7b5..d9d3c3567 100644 --- a/src/client/vfs_old/filesystem/defer_sync.cpp +++ b/src/client/vfs_old/filesystem/defer_sync.cpp @@ -28,14 +28,13 @@ #include #include "client/vfs_old/inode_wrapper.h" -#include "utils/concurrent/concurrent.h" #include "glog/logging.h" +#include "utils/concurrent/concurrent.h" namespace dingofs { namespace client { namespace filesystem { -using common::DeferSyncOption; using pb::metaserver::MetaStatusCode; using utils::LockGuard; using utils::Mutex; @@ -52,7 +51,7 @@ void SyncInodeClosure::Run() { defer_sync->Synced(sync_seq_, rc); } -DeferSync::DeferSync(DeferSyncOption option) +DeferSync::DeferSync(const DeferSyncOption& option) : option_(option), running_(false), sleeper_() {} void DeferSync::Start() { @@ -73,7 +72,7 @@ void DeferSync::Stop() { void DeferSync::SyncTask() { for (;;) { - bool running = sleeper_.wait_for(std::chrono::seconds(option_.delay)); + bool running = sleeper_.wait_for(std::chrono::seconds(option_.delay_s())); std::unordered_map> sync_inodes; { diff --git a/src/client/vfs_old/filesystem/defer_sync.h b/src/client/vfs_old/filesystem/defer_sync.h index e5b98d4b6..3fa41f322 100644 --- a/src/client/vfs_old/filesystem/defer_sync.h +++ b/src/client/vfs_old/filesystem/defer_sync.h @@ -29,9 +29,9 @@ #include #include -#include "client/common/config.h" #include "client/vfs_old/filesystem/meta.h" #include "client/vfs_old/inode_wrapper.h" +#include "options/client/vfs.h" #include "stub/rpcclient/task_excutor.h" #include "utils/interruptible_sleeper.h" @@ -39,6 +39,8 @@ namespace dingofs { namespace client { namespace filesystem { +using options::client::DeferSyncOption; + class DeferSync; class SyncInodeClosure : public stub::rpcclient::MetaServerClientDone { public: @@ -55,7 +57,7 @@ class SyncInodeClosure : public stub::rpcclient::MetaServerClientDone { class DeferSync : public std::enable_shared_from_this { public: - explicit DeferSync(common::DeferSyncOption option); + explicit DeferSync(const DeferSyncOption& option); void Start(); @@ -71,7 +73,7 @@ class DeferSync : public std::enable_shared_from_this { void SyncTask(); void Synced(uint64_t sync_seq, pb::metaserver::MetaStatusCode status); - common::DeferSyncOption option_; + DeferSyncOption option_; utils::Mutex mutex_; std::atomic running_; std::thread thread_; diff --git a/src/client/vfs_old/filesystem/dir_cache.cpp b/src/client/vfs_old/filesystem/dir_cache.cpp index 170f84ca5..d198ac900 100644 --- a/src/client/vfs_old/filesystem/dir_cache.cpp +++ b/src/client/vfs_old/filesystem/dir_cache.cpp @@ -24,9 +24,9 @@ #include -#include "dingofs/metaserver.pb.h" #include "base/time/time.h" #include "client/vfs_old/filesystem/utils.h" +#include "dingofs/metaserver.pb.h" namespace dingofs { namespace client { @@ -118,7 +118,7 @@ TimeSpec DirEntryList::GetMtime() { return mtime_; } -DirCache::DirCache(DirCacheOption option) +DirCache::DirCache(const DirCacheOption& option) : rwlock_(), nentries_(0), option_(option) { lru_ = std::make_shared(0); // control size by ourself mq_ = std::make_shared("dircache", 10000); @@ -126,14 +126,14 @@ DirCache::DirCache(DirCacheOption option) [&](const std::shared_ptr& entries) { entries->Clear(); }); metric_ = std::make_shared(); - LOG(INFO) << "Using directory lru cache, capacity = " << option_.lruSize; + LOG(INFO) << "Using directory lru cache, capacity = " << option_.lru_size(); } void DirCache::Start() { mq_->Start(); } void DirCache::Stop() { WriteLockGuard lk(rwlock_); - Evit(option_.lruSize); + Evit(option_.lru_size()); mq_->Stop(); } @@ -153,7 +153,7 @@ void DirCache::Delete(Ino parent, std::shared_ptr entries, void DirCache::Evit(size_t size) { Ino parent; std::shared_ptr entries; - while (nentries_ + size >= option_.lruSize) { + while (nentries_ + size >= option_.lru_size()) { bool yes = lru_->GetLast(&parent, &entries); if (!yes) { break; diff --git a/src/client/vfs_old/filesystem/dir_cache.h b/src/client/vfs_old/filesystem/dir_cache.h index d06ee9ade..2e81cc6d1 100644 --- a/src/client/vfs_old/filesystem/dir_cache.h +++ b/src/client/vfs_old/filesystem/dir_cache.h @@ -29,9 +29,9 @@ #include "absl/container/btree_map.h" #include "base/queue/message_queue.h" #include "base/time/time.h" -#include "client/common/config.h" #include "client/vfs_old/filesystem/meta.h" #include "client/vfs_old/filesystem/metric.h" +#include "options/client/vfs.h" #include "utils/concurrent/concurrent.h" #include "utils/concurrent/rw_lock.h" #include "utils/lru_cache.h" @@ -40,6 +40,8 @@ namespace dingofs { namespace client { namespace filesystem { +using options::client::DirCacheOption; + class DirEntryList { public: using IterateHandler = std::function; @@ -77,7 +79,7 @@ class DirCache { using MessageType = std::shared_ptr; using MessageQueueType = base::queue::MessageQueue; - explicit DirCache(common::DirCacheOption option); + explicit DirCache(const DirCacheOption& option); void Start(); @@ -96,7 +98,7 @@ class DirCache { utils::RWLock rwlock_; size_t nentries_; - common::DirCacheOption option_; + DirCacheOption option_; std::shared_ptr lru_; std::shared_ptr mq_; std::shared_ptr metric_; diff --git a/src/client/vfs_old/filesystem/dir_quota_manager.cpp b/src/client/vfs_old/filesystem/dir_quota_manager.cpp index 17ed221de..2adb7fc5b 100644 --- a/src/client/vfs_old/filesystem/dir_quota_manager.cpp +++ b/src/client/vfs_old/filesystem/dir_quota_manager.cpp @@ -22,6 +22,7 @@ #include "client/vfs_old/inode_wrapper.h" #include "common/define.h" #include "glog/logging.h" +#include "options/client/app.h" #include "utils/concurrent/concurrent.h" namespace dingofs { @@ -32,13 +33,11 @@ using utils::ReadLockGuard; using utils::RWLock; using utils::WriteLockGuard; +using options::client::OPTIONS_client; using pb::metaserver::MetaStatusCode; using pb::metaserver::Quota; using pb::metaserver::Usage; -USING_FLAG(flush_quota_interval_second); -USING_FLAG(load_quota_interval_second); - void DirQuota::UpdateUsage(int64_t new_space, int64_t new_inodes) { VLOG(6) << "UpdateUsage dir inodeId=" << ino_ << " new_space:" << new_space << ", new_inodes:" << new_inodes; @@ -137,10 +136,9 @@ void DirQuotaManager::Start() { return; } - timer_->Add([this] { FlushQuotas(); }, - FLAGS_flush_quota_interval_second * 1000); - timer_->Add([this] { LoadQuotas(); }, - FLAGS_load_quota_interval_second * 1000); + const auto& o = OPTIONS_client.vfs_option().quota_option(); + timer_->Add([this] { FlushQuotas(); }, o.flush_quota_interval_s() * 1000); + timer_->Add([this] { LoadQuotas(); }, o.load_quota_interval_s() * 1000); running_.store(true); } diff --git a/src/client/vfs_old/filesystem/filesystem.cpp b/src/client/vfs_old/filesystem/filesystem.cpp index 152f30708..3f1e0fce0 100644 --- a/src/client/vfs_old/filesystem/filesystem.cpp +++ b/src/client/vfs_old/filesystem/filesystem.cpp @@ -44,7 +44,6 @@ namespace filesystem { using base::time::TimeSpec; using base::timer::TimerImpl; -using common::FileSystemOption; using pb::metaserver::InodeAttr; using pb::metaserver::Quota; @@ -55,15 +54,15 @@ using ::dingofs::client::common::ShareVar; USING_FLAG(stat_timer_thread_num); FileSystem::FileSystem(uint32_t fs_id, std::string fs_name, - FileSystemOption option, ExternalMember member) + const VFSOption& option, ExternalMember member) : fs_id_(fs_id), fs_name_(fs_name), option_(option), member(member) { - deferSync_ = std::make_shared(option.deferSyncOption); - negative_ = std::make_shared(option.lookupCacheOption); - dirCache_ = std::make_shared(option.dirCacheOption); + deferSync_ = std::make_shared(option.defer_sync_option()); + negative_ = std::make_shared(option.lookup_cache_option()); + dirCache_ = std::make_shared(option.dir_cache_option()); openFiles_ = std::make_shared(option_.openFilesOption, deferSync_); - attrWatcher_ = std::make_shared(option_.attrWatcherOption, + attrWatcher_ = std::make_shared(option_.attr_watcher_option(), openFiles_, dirCache_); - entry_watcher_ = std::make_shared(option_.nocto_suffix); + entry_watcher_ = std::make_shared(option_.nocto_suffix()); handlerManager_ = std::make_shared(); rpc_ = std::make_shared(option.rpcOption, member); } diff --git a/src/client/vfs_old/filesystem/filesystem.h b/src/client/vfs_old/filesystem/filesystem.h index 185500c35..95bd9248a 100644 --- a/src/client/vfs_old/filesystem/filesystem.h +++ b/src/client/vfs_old/filesystem/filesystem.h @@ -30,7 +30,6 @@ #include #include "base/timer/timer.h" -#include "client/common/config.h" #include "client/vfs_old/filesystem/attr_watcher.h" #include "client/vfs_old/filesystem/defer_sync.h" #include "client/vfs_old/filesystem/dir_cache.h" @@ -45,6 +44,7 @@ #include "client/vfs_old/filesystem/openfile.h" #include "client/vfs_old/filesystem/package.h" #include "client/vfs_old/filesystem/rpc_client.h" +#include "options/client/vfs.h" namespace dingofs { namespace client { @@ -56,6 +56,8 @@ class VFSOld; namespace filesystem { +using options::client::VFSOption; + struct FileSystemMember { FileSystemMember(std::shared_ptr deferSync, std::shared_ptr openFiles, @@ -74,8 +76,8 @@ struct FileSystemMember { class FileSystem { public: - FileSystem(uint32_t fs_id, std::string fs_name, - common::FileSystemOption option, ExternalMember member); + FileSystem(uint32_t fs_id, std::string fs_name, const VFSOption& option, + ExternalMember member); void Run(); @@ -150,7 +152,7 @@ class FileSystem { uint32_t fs_id_; std::string fs_name_; - common::FileSystemOption option_; + VFSOption option_; ExternalMember member; std::shared_ptr deferSync_; std::shared_ptr negative_; diff --git a/src/client/vfs_old/filesystem/lookup_cache.cpp b/src/client/vfs_old/filesystem/lookup_cache.cpp index 221ce7e1b..600bc466b 100644 --- a/src/client/vfs_old/filesystem/lookup_cache.cpp +++ b/src/client/vfs_old/filesystem/lookup_cache.cpp @@ -46,13 +46,13 @@ using utils::WriteLockGuard; } \ } while (0) -LookupCache::LookupCache(LookupCacheOption option) - : enable_(option.negativeTimeoutSec > 0), rwlock_(), option_(option) { - lru_ = std::make_shared(option.lruSize); +LookupCache::LookupCache(const LookupCacheOption& option) + : enable_(option.negative_timeout_s() > 0), rwlock_(), option_(option) { + lru_ = std::make_shared(option.lru_size()); if (enable_) { LOG(INFO) << "Using lookup negative lru cache" - << ", timeout = " << option.negativeTimeoutSec - << ", capacity = " << option.lruSize; + << ", timeout = " << option.negative_timeout_s() + << ", capacity = " << option.lru_size(); } } @@ -70,7 +70,7 @@ bool LookupCache::Get(Ino parent, const std::string& name) { VLOG(1) << absl::StrFormat("Lookup cache not found: key(%d,%s)", parent, name); return false; - } else if (entry.uses < option_.minUses) { + } else if (entry.uses < option_.min_uses()) { return false; } else if (entry.expireTime < Now()) { return false; @@ -91,7 +91,7 @@ bool LookupCache::Put(Ino parent, const std::string& name) { } entry.expireTime = - Now() + base::time::TimeSpec(option_.negativeTimeoutSec, 0); + Now() + base::time::TimeSpec(option_.negative_timeout_s(), 0); lru_->Put(key, entry); return true; } diff --git a/src/client/vfs_old/filesystem/lookup_cache.h b/src/client/vfs_old/filesystem/lookup_cache.h index 247a54225..dd56f185e 100644 --- a/src/client/vfs_old/filesystem/lookup_cache.h +++ b/src/client/vfs_old/filesystem/lookup_cache.h @@ -26,14 +26,16 @@ #include #include -#include "client/common/config.h" #include "client/vfs_old/filesystem/meta.h" +#include "options/client/vfs.h" #include "utils/lru_cache.h" namespace dingofs { namespace client { namespace filesystem { +using options::client::LookupCacheOption; + // memory cache for lookup result, now we only support cache negative result, // and other positive entry will be cached in kernel. class LookupCache { @@ -45,7 +47,7 @@ class LookupCache { using LRUType = utils::LRUCache; - explicit LookupCache(common::LookupCacheOption option); + explicit LookupCache(const LookupCacheOption& option); bool Get(Ino parent, const std::string& name); @@ -58,7 +60,7 @@ class LookupCache { bool enable_; utils::RWLock rwlock_; - common::LookupCacheOption option_; + LookupCacheOption option_; std::shared_ptr lru_; }; diff --git a/src/client/vfs_old/filesystem/openfile.cpp b/src/client/vfs_old/filesystem/openfile.cpp index 56ee4187d..bc774319f 100644 --- a/src/client/vfs_old/filesystem/openfile.cpp +++ b/src/client/vfs_old/filesystem/openfile.cpp @@ -31,16 +31,13 @@ namespace dingofs { namespace client { namespace filesystem { -using common::OpenFilesOption; using utils::ReadLockGuard; using utils::WriteLockGuard; -OpenFiles::OpenFiles(OpenFilesOption option, - std::shared_ptr defer_sync) - : option_(option), deferSync_(defer_sync) { +OpenFiles::OpenFiles(std::shared_ptr defer_sync) + : deferSync_(defer_sync) { metric_ = std::make_shared(); - LOG(INFO) << "Using openfile lru cache but ignored, capacity is " - << option.lruSize; + LOG(INFO) << "Using openfile cache."; } void OpenFiles::Open(Ino ino, std::shared_ptr inode) { diff --git a/src/client/vfs_old/filesystem/openfile.h b/src/client/vfs_old/filesystem/openfile.h index 0c0e11905..77dac7b62 100644 --- a/src/client/vfs_old/filesystem/openfile.h +++ b/src/client/vfs_old/filesystem/openfile.h @@ -47,8 +47,7 @@ class OpenFiles { public: // option not used, but keeped here, we can turn on/off openfiles cache in the // future like juicefs - explicit OpenFiles(common::OpenFilesOption option, - std::shared_ptr defer_sync); + explicit OpenFiles(std::shared_ptr defer_sync); void Open(Ino ino, std::shared_ptr inode); @@ -62,7 +61,6 @@ class OpenFiles { private: utils::RWLock rwlock_; - common::OpenFilesOption option_; std::shared_ptr deferSync_; std::unordered_map> files_; std::shared_ptr metric_; diff --git a/src/client/vfs_old/kvclient/kvclient.h b/src/client/vfs_old/kvclient/kvclient.h deleted file mode 100644 index 63c6592a8..000000000 --- a/src/client/vfs_old/kvclient/kvclient.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2022 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Project: dingo - * Created Date: 2022-09-22 - * Author: YangFan (fansehep) - */ -#ifndef DINGOFS_SRC_CLIENT_KVCLIENT_KVCLIENT_H_ -#define DINGOFS_SRC_CLIENT_KVCLIENT_KVCLIENT_H_ - -#include - -namespace dingofs { - -namespace client { - -/** - * Single client to kv interface. - */ - -class KVClient { - public: - KVClient() = default; - ~KVClient() = default; - - virtual void Init() {} - - virtual void UnInit() {} - - /** - * @param: errorlog: if error occurred, the errorlog will take - * the error info and log. - * @return: success return true, else return false; - */ - virtual bool Set(const std::string& key, const char* value, - const uint64_t value_len, std::string* errorlog) = 0; - - virtual bool Get(const std::string& key, char* value, uint64_t offset, - uint64_t length, std::string* errorlog) = 0; -}; - -} // namespace client -} // namespace dingofs -#endif // DINGOFS_SRC_CLIENT_KVCLIENT_KVCLIENT_H_ diff --git a/src/client/vfs_old/kvclient/kvclient_manager.cpp b/src/client/vfs_old/kvclient/kvclient_manager.cpp deleted file mode 100644 index 6b3c9c361..000000000 --- a/src/client/vfs_old/kvclient/kvclient_manager.cpp +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2022 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Project: dingo - * Created Date: 2022-10-18 - * Author: lixiaocui - */ - -#include "client/vfs_old/kvclient/kvclient_manager.h" - -#include "stub/metric/metric.h" - -using dingofs::stub::metric::LatencyGuard; - -namespace dingofs { -namespace client { - -using common::KVClientManagerOpt; - -#define ONRETURN(TYPE, RES) \ - if (RES) { \ - kvClientMetric_.kvClient##TYPE.qps.count << 1; \ - } else { \ - kvClientMetric_.kvClient##TYPE.eps.count << 1; \ - } - -bool KVClientManager::Init(const KVClientManagerOpt& config, - const std::shared_ptr& kvclient) { - client_ = kvclient; - return threadPool_.Start(config.setThreadPooln) == 0; -} - -void KVClientManager::Uninit() { - client_->UnInit(); - threadPool_.Stop(); -} - -void KVClientManager::Set(std::shared_ptr task) { - threadPool_.Enqueue([task, this]() { - LatencyGuard guard(&kvClientMetric_.kvClientSet.latency); - - std::string error_log; - auto res = client_->Set(task->key, task->value, task->length, &error_log); - ONRETURN(Set, res); - - task->done(task); - }); -} - -void KVClientManager::Get(std::shared_ptr task) { - threadPool_.Enqueue([task, this]() { - LatencyGuard guard(&kvClientMetric_.kvClientGet.latency); - - std::string error_log; - task->res = client_->Get(task->key, task->value, task->offset, task->length, - &error_log); - ONRETURN(Get, task->res); - - task->done(task); - }); -} - -} // namespace client -} // namespace dingofs diff --git a/src/client/vfs_old/kvclient/kvclient_manager.h b/src/client/vfs_old/kvclient/kvclient_manager.h deleted file mode 100644 index 03e3886e4..000000000 --- a/src/client/vfs_old/kvclient/kvclient_manager.h +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright (c) 2022 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Project: dingo - * Created Date: 2022-09-23 - * Author: YangFan (fansehep) - */ - -#ifndef DINGOFS_SRC_CLIENT_KVCLIENT_KVCLIENT_MANAGER_H_ -#define DINGOFS_SRC_CLIENT_KVCLIENT_KVCLIENT_MANAGER_H_ - -#include - -#include -#include -#include - -#include "client/common/config.h" -#include "client/vfs_old/kvclient/kvclient.h" -#include "stub/metric/metric.h" -#include "utils/concurrent/task_thread_pool.h" - -namespace dingofs { -namespace client { - -class KVClientManager; -class SetKVCacheTask; -class GetKVCacheTask; - -using SetKVCacheDone = - std::function&)>; -using GetKVCacheDone = - std::function&)>; - -struct SetKVCacheTask { - std::string key; - const char* value; - uint64_t length; - SetKVCacheDone done; - SetKVCacheTask() = default; - SetKVCacheTask( - const std::string& k, const char* val, const uint64_t len, - SetKVCacheDone done = [](const std::shared_ptr&) {}) - : key(k), value(val), length(len), done(std::move(done)) {} -}; - -struct GetKVCacheTask { - const std::string& key; - char* value; - uint64_t offset; - uint64_t length; - bool res; - GetKVCacheDone done; - GetKVCacheTask(const std::string& k, char* v, uint64_t off, uint64_t len) - : key(k), value(v), offset(off), length(len), res(false) { - done = [](const std::shared_ptr&) {}; - } -}; - -class KVClientManager { - public: - KVClientManager() = default; - ~KVClientManager() { Uninit(); } - - bool Init(const common::KVClientManagerOpt& config, - const std::shared_ptr& kvclient); - - /** - * It will get a db client and set the key value asynchronusly. - * The set task will push threadpool, you'd better - * don't get the key immediately. - */ - void Set(std::shared_ptr task); - - void Get(std::shared_ptr task); - - stub::metric::KVClientMetric* GetClientMetricForTesting() { - return &kvClientMetric_; - } - - private: - void Uninit(); - - utils::TaskThreadPool threadPool_; - std::shared_ptr client_; - stub::metric::KVClientMetric kvClientMetric_; -}; - -} // namespace client -} // namespace dingofs -#endif // DINGOFS_SRC_CLIENT_KVCLIENT_KVCLIENT_MANAGER_H_ diff --git a/src/client/vfs_old/kvclient/memcache_client.cpp b/src/client/vfs_old/kvclient/memcache_client.cpp deleted file mode 100644 index d04f7a678..000000000 --- a/src/client/vfs_old/kvclient/memcache_client.cpp +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2022 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Project: dingo - * Created Date: 2022-10-08 - * Author: YangFan (fansehep) - */ - -#include "client/vfs_old/kvclient/memcache_client.h" - -namespace dingofs { -namespace client { - -thread_local memcached_st* tcli = nullptr; - -} // namespace client -} // namespace dingofs diff --git a/src/client/vfs_old/kvclient/memcache_client.h b/src/client/vfs_old/kvclient/memcache_client.h deleted file mode 100644 index 53994460f..000000000 --- a/src/client/vfs_old/kvclient/memcache_client.h +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Copyright (c) 2022 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Project: dingo - * Created Date: 2022-09-25 - * Author: YangFan (fansehep) - */ - -#ifndef DINGOFS_SRC_CLIENT_KVCLIENT_MEMCACHE_CLIENT_H_ -#define DINGOFS_SRC_CLIENT_KVCLIENT_MEMCACHE_CLIENT_H_ - -#include -#include -#include - -#include - -#include "dingofs/topology.pb.h" -#include "client/vfs_old/kvclient/kvclient.h" - -namespace dingofs { - -namespace client { - -/** - * only the threadpool will operate the kvclient, - * for threadsafe and fast, we can make every thread has a client. - */ -extern thread_local memcached_st* tcli; - -/** - * MemCachedClient is a client to memcached cluster. You'd better - * don't use it directly. - * normal usage example: - * auto client = std::make_unique(); - * (make_unique is after c++14) - * std::string error_log; - * auto ue = client->AddServer("x.x.x.x", port); - * ... (the number of your memcached cluster) - * if (!ue) {...} - * ue = client->PushServer(); - * if (!ue) {...} - * uint64_t data; - * client->SetClientAttr(MEMCACHED_BEHAVIOR_NO_BLOCK, data); - * client->SetClientAttr(MEMCACHED_BEHAVIOR_TCP_NODELAY, data); - * KVClientManager manager; - * config.kvclient = std::move(client_); - * config.threadPooln = n; - * manager.Init(&conf); - * char key[] = "dingo"; - * char value[] = "yyds" - * manager.Set(key, value, strlen(value)); - * ... (don't get it immediately, the kvmanager has a threadpool - * it will perform tasks asynchronously) - * std::string res; - * ue = manager.Get(key, &res); - * if (!ue) {...} - * then ... - * manager.Unint(); - */ - -class MemCachedClient : public KVClient { - public: - MemCachedClient() : server_(nullptr) { client_ = memcached_create(nullptr); } - explicit MemCachedClient(memcached_st* cli) : client_(cli) {} - ~MemCachedClient() { UnInit(); } - - bool Init(const pb::mds::topology::MemcacheClusterInfo& kvcachecluster) { - client_ = memcached(nullptr, 0); - - for (int i = 0; i < kvcachecluster.servers_size(); i++) { - if (!AddServer(kvcachecluster.servers(i).ip(), - kvcachecluster.servers(i).port())) { - return false; - } - } - memcached_behavior_set(client_, MEMCACHED_BEHAVIOR_DISTRIBUTION, - MEMCACHED_DISTRIBUTION_CONSISTENT); - memcached_behavior_set(client_, MEMCACHED_BEHAVIOR_RETRY_TIMEOUT, 5); - - return PushServer(); - } - - void UnInit() override { - if (client_) { - memcached_free(client_); - client_ = nullptr; - } - } - - bool Set(const std::string& key, const char* value, const uint64_t value_len, - std::string* errorlog) override { - if (nullptr == tcli) { - tcli = memcached_clone(nullptr, client_); - } - auto res = - memcached_set(tcli, key.c_str(), key.length(), value, value_len, 0, 0); - if (MEMCACHED_SUCCESS == res) { - VLOG(9) << "Set key = " << key << " OK"; - return true; - } - *errorlog = ResError(res); - memcached_free(tcli); - tcli = nullptr; - LOG(ERROR) << "Set key = " << key << " error = " << *errorlog; - return false; - } - - bool Get(const std::string& key, char* value, uint64_t offset, - uint64_t length, std::string* errorlog) override { - if (nullptr == tcli) { - // multi thread use a memcached_st* client is unsafe. - // should clone it or use memcached_st_pool. - tcli = memcached_clone(nullptr, client_); - } - uint32_t flags = 0; - size_t value_length = 0; - memcached_return_t ue; - char* res = memcached_get(tcli, key.c_str(), key.length(), &value_length, - &flags, &ue); - if (MEMCACHED_SUCCESS == ue && res != nullptr && value && - value_length >= length) { - VLOG(9) << "Get key = " << key << " OK"; - memcpy(value, res + offset, length); - free(res); - return true; - } - - *errorlog = ResError(ue); - if (ue != MEMCACHED_NOTFOUND) { - LOG(ERROR) << "Get key = " << key << " error = " << *errorlog - << ", get_value_len = " << value_length - << ", expect_value_len = " << length; - memcached_free(tcli); - tcli = nullptr; - } - - return false; - } - - // transform the res to a error string - const std::string ResError(const memcached_return_t res) { - return memcached_strerror(nullptr, res); - } - - /** - * @brief: add a remote memcache server to client, - * this means just add, you must use push after all server add. - */ - bool AddServer(const std::string& hostname, const uint32_t port) { - memcached_return_t res; - server_ = - memcached_server_list_append(server_, hostname.c_str(), port, &res); - if (MEMCACHED_SUCCESS == res) { - return true; - } - LOG(ERROR) << "client add " << hostname << " " << port << " error"; - return false; - } - - /** - * @brief: push the server list to the client - */ - bool PushServer() { - memcached_return_t res = memcached_server_push(client_, server_); - if (MEMCACHED_SUCCESS == res) { - return true; - } - memcached_server_list_free(server_); - server_ = nullptr; - return false; - } - - /** - * @return: return this client number of remote servers - */ - int ServerCount() { - return static_cast(memcached_server_count(client_)); - } - - private: - using KVClient::Init; - memcached_server_st* server_; - memcached_st* client_; -}; - -} // namespace client -} // namespace dingofs -#endif // DINGOFS_SRC_CLIENT_KVCLIENT_MEMCACHE_CLIENT_H_ diff --git a/src/client/vfs_old/vfs_old.cpp b/src/client/vfs_old/vfs_old.cpp index ea4af952f..23e06a77e 100644 --- a/src/client/vfs_old/vfs_old.cpp +++ b/src/client/vfs_old/vfs_old.cpp @@ -80,6 +80,10 @@ static void OnThrottleTimer(void* arg) { } void VFSOld::InitQosParam() { + const auto& o = option_.vfs_option().throttle_option(); + + // o.avg_write_iops(), + utils::ReadWriteThrottleParams params; params.iopsWrite = utils::ThrottleParams(common::FLAGS_fuseClientAvgWriteIops, @@ -150,8 +154,8 @@ int VFSOld::InitBrpcServer() { uint32_t listen_port = 0; if (!StartBrpcServer(server_, &brpc_server_options, - fuse_client_option_.dummyServerStartPort, PORT_LIMIT, - &listen_port)) { + option_.global_option().dummy_server_start_port(), + PORT_LIMIT, &listen_port)) { LOG(ERROR) << "Start brpc server failed!"; return -1; } @@ -444,22 +448,20 @@ Status VFSOld::Stop() { } double VFSOld::GetAttrTimeout(const FileType& type) { + const auto& option = option_.kernel_cache_option(); if (type == FileType::kDirectory) { - return fuse_client_option_.fileSystemOption.kernelCacheOption - .dirAttrTimeoutSec; + return option.dir_attr_timeout_s(); } else { - return fuse_client_option_.fileSystemOption.kernelCacheOption - .attrTimeoutSec; + return option.attr_timeout_s(); } } double VFSOld::GetEntryTimeout(const FileType& type) { + const auto& option = option_.kernel_cache_option(); if (type == FileType::kDirectory) { - return fuse_client_option_.fileSystemOption.kernelCacheOption - .dirEntryTimeoutSec; + return option.dir_entry_timeout_s(); } else { - return fuse_client_option_.fileSystemOption.kernelCacheOption - .entryTimeoutSec; + return option.entry_timeout_s(); } } diff --git a/src/client/vfs_old/vfs_old.h b/src/client/vfs_old/vfs_old.h index 69247464f..74d17c9c9 100644 --- a/src/client/vfs_old/vfs_old.h +++ b/src/client/vfs_old/vfs_old.h @@ -25,7 +25,6 @@ #include #include "client/common/common.h" -#include "client/common/config.h" #include "client/common/status.h" #include "client/vfs/vfs.h" #include "client/vfs/vfs_meta.h" @@ -34,6 +33,8 @@ #include "client/vfs_old/service/inode_objects_service.h" #include "client/vfs_old/warmup/warmup_manager.h" #include "dingofs/mds.pb.h" +#include "options/client/app.h" +#include "options/client/vfs.h" #include "stub/rpcclient/mds_client.h" #include "utils/throttle.h" @@ -41,10 +42,11 @@ namespace dingofs { namespace client { namespace vfs { +using options::client::VFSOption; + class VFSOld : public VFS { public: - VFSOld(const common::ClientOption& fuse_client_option) - : fuse_client_option_(fuse_client_option) {} + VFSOld(const VFSOption& option) : option_(option) {} ~VFSOld() override = default; @@ -122,10 +124,6 @@ class VFSOld : public VFS { uint64_t GetMaxNameLength() override; - common::FuseOption GetFuseOption() override { - return fuse_client_option_.fuse_option; - } - void InitQosParam(); private: @@ -163,7 +161,7 @@ class VFSOld : public VFS { pb::mds::Mountpoint mount_point_; - common::ClientOption fuse_client_option_; + VFSOption option_; // fs info std::shared_ptr fs_info_{nullptr}; diff --git a/src/client/vfs_wrapper/global_log.h b/src/client/vfs_wrapper/global_log.h index 44341bc53..c3bc0c815 100644 --- a/src/client/vfs_wrapper/global_log.h +++ b/src/client/vfs_wrapper/global_log.h @@ -15,31 +15,24 @@ */ #include "common/dynamic_vlog.h" +#include "options/client/app.h" #include "utils/configuration.h" #include "utils/gflags_helper.h" static int InitLog(const char* argv0, std::string conf_path) { - dingofs::utils::Configuration conf; - conf.SetConfigPath(conf_path); - if (!conf.LoadConfig()) { - LOG(ERROR) << "LoadConfig failed, confPath = " << conf_path; - return 1; - } + dingofs::options::client::AppOption option; - // set log dir - if (FLAGS_log_dir.empty()) { - if (!conf.GetStringValue("client.common.logDir", &FLAGS_log_dir)) { - LOG(WARNING) << "no client.common.logDir in " << conf_path - << ", will log to /tmp"; - } + if (!option.Parse(conf_path)) { + LOG(ERROR) << "Parse config file failed, confpath = " << conf_path; + return 1; } - dingofs::utils::GflagsLoadValueFromConfIfCmdNotSet dummy; - dummy.Load(&conf, "v", "client.loglevel", &FLAGS_v); - dingofs::common::FLAGS_vlog_level = FLAGS_v; + const auto& global_option = option.global_option(); - FLAGS_logbufsecs = 0; // initialize logging module + FLAGS_log_dir = global_option.log_dir(); + FLAGS_v = global_option.vlog_level(); + FLAGS_logbufsecs = 0; google::InitGoogleLogging(argv0); return 0; diff --git a/src/client/vfs_wrapper/vfs_wrapper.cpp b/src/client/vfs_wrapper/vfs_wrapper.cpp index f7dfcc91c..40ac1f159 100644 --- a/src/client/vfs_wrapper/vfs_wrapper.cpp +++ b/src/client/vfs_wrapper/vfs_wrapper.cpp @@ -32,6 +32,7 @@ #include "client/vfs_wrapper/access_log.h" #include "common/rpc_stream.h" #include "dataaccess/aws/s3_access_log.h" +#include "options/client/app.h" #include "stub/metric/metric.h" #include "stub/rpcclient/meta_access_log.h" #include "utils/configuration.h" @@ -40,6 +41,8 @@ namespace dingofs { namespace client { namespace vfs { +using options::client::OPTIONS_client; + #define METRIC_GUARD(REQUEST) \ ClientOpMetricGuard clientOpMetricGuard( \ &rc, {&client_op_metric_->op##REQUEST, &client_op_metric_->opAll}); @@ -68,14 +71,6 @@ Status InitLog() { return Status::OK(); } -static Status InitConfig(utils::Configuration& conf, - common::ClientOption& fuse_client_option) { - // init fuse client option - common::InitClientOption(&conf, &fuse_client_option); - - return Status::OK(); -} - Status VFSWrapper::Start(const char* argv0, const VFSConfig& vfs_conf) { VLOG(1) << "VFSStart argv0: " << argv0; @@ -93,15 +88,11 @@ Status VFSWrapper::Start(const char* argv0, const VFSConfig& vfs_conf) { AccessLogGuard log( [&]() { return absl::StrFormat("start: %s", s.ToString()); }); - // load config - s = LoadConfig(vfs_conf.config_path, conf_); - if (!s.ok()) { - return s; + bool succ = OPTIONS_client.Parse(vfs_conf.config_path); + if (!succ) { + return Status::Internal("parse config file failed"); } - // init client option - common::InitClientOption(&conf_, &fuse_client_option_); - // init log s = InitLog(); if (!s.ok()) { @@ -109,7 +100,7 @@ Status VFSWrapper::Start(const char* argv0, const VFSConfig& vfs_conf) { } int32_t bthread_worker_num = - dingofs::client::common::FLAGS_bthread_worker_num; + OPTIONS_client.global_option().bthread_worker_num(); if (bthread_worker_num > 0) { bthread_setconcurrency(bthread_worker_num); LOG(INFO) << "set bthread concurrency to " << bthread_worker_num @@ -121,10 +112,10 @@ Status VFSWrapper::Start(const char* argv0, const VFSConfig& vfs_conf) { client_op_metric_ = std::make_unique(); if (vfs_conf.fs_type == "vfs" || vfs_conf.fs_type == "vfs_v1" || vfs_conf.fs_type == "vfs_v2" || vfs_conf.fs_type == "vfs_dummy") { - vfs_ = std::make_unique(fuse_client_option_); + vfs_ = std::make_unique(option_); } else { - vfs_ = std::make_unique(fuse_client_option_); + vfs_ = std::make_unique(option_); } return vfs_->Start(vfs_conf); @@ -169,7 +160,7 @@ Status VFSWrapper::Lookup(Ino parent, const std::string& name, Attr* attr) { ClientOpMetricGuard op_metric( {&client_op_metric_->opLookup, &client_op_metric_->opAll}); - if (name.length() > fuse_client_option_.fileSystemOption.maxNameLength) { + if (name.length() > option) { s = Status::NameTooLong(fmt::format("name({}) too long", name.length())); return s; } @@ -728,10 +719,6 @@ uint64_t VFSWrapper::GetMaxNameLength() { return max_name_length; } -common::FuseOption VFSWrapper::GetFuseOption() const { - return vfs_->GetFuseOption(); -} - } // namespace vfs } // namespace client } // namespace dingofs \ No newline at end of file diff --git a/src/client/vfs_wrapper/vfs_wrapper.h b/src/client/vfs_wrapper/vfs_wrapper.h index f72b5030f..e5fb01830 100644 --- a/src/client/vfs_wrapper/vfs_wrapper.h +++ b/src/client/vfs_wrapper/vfs_wrapper.h @@ -20,8 +20,8 @@ #include #include -#include "client/common/config.h" #include "client/vfs/vfs.h" +#include "options/client/app.h" #include "stub/metric/metric.h" namespace dingofs { @@ -118,12 +118,7 @@ class VFSWrapper { uint64_t GetMaxNameLength(); - common::FuseOption GetFuseOption() const; - private: - utils::Configuration conf_; - common::ClientOption fuse_client_option_; - std::unique_ptr vfs_; std::unique_ptr client_op_metric_; }; diff --git a/src/dataaccess/README.md b/src/dataaccess/README.md new file mode 100644 index 000000000..ebf78c0d3 --- /dev/null +++ b/src/dataaccess/README.md @@ -0,0 +1,15 @@ +Data Access +=== + +DataAccesser is a class that provides a way to access data from a data source. It is a base class for all data access classes. + +Module Level +--- + +``` + +-- S3 Accesser -> S3 Adapter -> { aws_crt_client / aws_lagacy_client } + | +Data Accesser + + | + +--> Rados Accesser +``` diff --git a/src/dataaccess/aws/CMakeLists.txt b/src/dataaccess/aws/CMakeLists.txt index c8e172486..1753c290f 100644 --- a/src/dataaccess/aws/CMakeLists.txt +++ b/src/dataaccess/aws/CMakeLists.txt @@ -21,6 +21,7 @@ add_library(aws_s3_adapter target_link_libraries(aws_s3_adapter dingofs_utils + dingofs_options ${AWSSDK_LIBRARIES} glog::glog brpc::brpc diff --git a/src/dataaccess/aws/aws_s3_common.h b/src/dataaccess/aws/aws_s3_common.h index 8386158aa..8022e8d70 100644 --- a/src/dataaccess/aws/aws_s3_common.h +++ b/src/dataaccess/aws/aws_s3_common.h @@ -24,6 +24,8 @@ #include #include "dataaccess/accesser_common.h" +#include "options/client/s3.h" +#include "options/options.h" #include "utils/configuration.h" #include "utils/macros.h" @@ -33,98 +35,72 @@ namespace dingofs { namespace dataaccess { namespace aws { -struct S3AdapterOption { - std::string ak; - std::string sk; - std::string s3Address; - std::string bucketName; - std::string region{"us-east-1"}; - int loglevel{4}; - std::string logPrefix; - bool verifySsl{false}; - int maxConnections{32}; - int connectTimeout{60000}; - int requestTimeout{10000}; - bool use_crt_client{false}; - bool use_thread_pool{true}; // this only work when use_crt_client is false - int asyncThreadNum{16}; // this only work when use_crt_client is false - uint64_t maxAsyncRequestInflightBytes{0}; - uint64_t iopsTotalLimit{0}; - uint64_t iopsReadLimit{0}; - uint64_t iopsWriteLimit{0}; - uint64_t bpsTotalMB{0}; - uint64_t bpsReadMB{0}; - uint64_t bpsWriteMB{0}; - bool useVirtualAddressing{false}; - bool enableTelemetry{false}; -}; - -struct S3InfoOption { - // should get from mds - std::string ak; - std::string sk; - std::string s3Address; - std::string bucketName; - uint64_t blockSize; - uint64_t chunkSize; - uint32_t objectPrefix; -}; +using options::client::S3Option; inline void InitS3AdaptorOptionExceptS3InfoOption(utils::Configuration* conf, - S3AdapterOption* s3_opt) { - LOG_IF(FATAL, !conf->GetIntValue("s3.logLevel", &s3_opt->loglevel)); - LOG_IF(FATAL, !conf->GetStringValue("s3.logPrefix", &s3_opt->logPrefix)); - LOG_IF(FATAL, !conf->GetBoolValue("s3.verify_SSL", &s3_opt->verifySsl)); - LOG_IF(FATAL, - !conf->GetIntValue("s3.maxConnections", &s3_opt->maxConnections)); - LOG_IF(FATAL, - !conf->GetIntValue("s3.connectTimeout", &s3_opt->connectTimeout)); + S3Option* s3_option) { + auto* global_option = &s3_option->global_option(); + auto* request_option = &s3_option->request_option(); + auto* throttle_option = &s3_option->throttle_option(); + + // global + LOG_IF(FATAL, !conf->GetIntValue("s3.logLevel", &global_option->log_level())); LOG_IF(FATAL, - !conf->GetIntValue("s3.requestTimeout", &s3_opt->requestTimeout)); + !conf->GetStringValue("s3.logPrefix", &global_option->log_prefix())); + if (!conf->GetBoolValue("s3.enableTelemetry", + &global_option->telemetry_enable())) { + LOG(WARNING) << "Not found s3.enableTelemetry in conf,default to false"; + global_option->telemetry_enable() = false; + } - if (!conf->GetBoolValue("s3.use_crt_client", &s3_opt->use_crt_client)) { - s3_opt->use_crt_client = false; + // request + LOG_IF(FATAL, + !conf->GetBoolValue("s3.verify_SSL", &request_option->verify_ssl())); + LOG_IF(FATAL, !conf->GetIntValue("s3.maxConnections", + &request_option->max_connections())); + LOG_IF(FATAL, !conf->GetIntValue("s3.connectTimeout", + &request_option->connect_timeout_ms())); + LOG_IF(FATAL, !conf->GetIntValue("s3.requestTimeout", + &request_option->request_timeout_ms())); + + if (!conf->GetBoolValue("s3.use_crt_client", + &request_option->use_crt_client())) { LOG(INFO) << "Not found s3.use_crt_client in conf, use default " - << (s3_opt->use_crt_client ? "true" : "false"); + << (request_option->use_crt_client() ? "true" : "false"); } - - if (!conf->GetBoolValue("s3.use_thread_pool", &s3_opt->use_thread_pool)) { + if (!conf->GetBoolValue("s3.use_thread_pool", + &request_option->use_thread_pool())) { LOG(INFO) << "Not found s3.use_thread_pool in conf, use default " - << (s3_opt->use_thread_pool ? "true" : "false"); + << (request_option->use_thread_pool() ? "true" : "false"); } - if (!conf->GetIntValue("s3.async_thread_num_in_thread_pool", - &s3_opt->asyncThreadNum)) { + &request_option->async_thread_num())) { LOG(INFO) << "Not found s3.async_thread_num_in_thread_pool in conf, use default" - << s3_opt->asyncThreadNum; + << request_option->async_thread_num(); + } + if (!conf->GetUInt64Value( + "s3.maxAsyncRequestInflightBytes", + &request_option->max_async_request_inflight_bytes())) { + LOG(WARNING) << "Not found s3.maxAsyncRequestInflightBytes in conf"; } + // throttle LOG_IF(FATAL, !conf->GetUInt64Value("s3.throttle.iopsTotalLimit", - &s3_opt->iopsTotalLimit)); + &throttle_option->iops_total_limit())); LOG_IF(FATAL, !conf->GetUInt64Value("s3.throttle.iopsReadLimit", - &s3_opt->iopsReadLimit)); + &throttle_option->iops_read_limit())); LOG_IF(FATAL, !conf->GetUInt64Value("s3.throttle.iopsWriteLimit", - &s3_opt->iopsWriteLimit)); - LOG_IF(FATAL, - !conf->GetUInt64Value("s3.throttle.bpsTotalMB", &s3_opt->bpsTotalMB)); - LOG_IF(FATAL, - !conf->GetUInt64Value("s3.throttle.bpsReadMB", &s3_opt->bpsReadMB)); - LOG_IF(FATAL, - !conf->GetUInt64Value("s3.throttle.bpsWriteMB", &s3_opt->bpsWriteMB)); + &throttle_option->iops_write_limit())); + LOG_IF(FATAL, !conf->GetUInt64Value("s3.throttle.bpsTotalMB", + &throttle_option->bps_total_mb())); + LOG_IF(FATAL, !conf->GetUInt64Value("s3.throttle.bpsReadMB", + &throttle_option->bps_read_mb())); + LOG_IF(FATAL, !conf->GetUInt64Value("s3.throttle.bpsWriteMB", + &throttle_option->bps_write_mb())); LOG_IF(FATAL, !conf->GetBoolValue("s3.useVirtualAddressing", - &s3_opt->useVirtualAddressing)); - LOG_IF(FATAL, !conf->GetStringValue("s3.region", &s3_opt->region)); - - if (!conf->GetUInt64Value("s3.maxAsyncRequestInflightBytes", - &s3_opt->maxAsyncRequestInflightBytes)) { - LOG(WARNING) << "Not found s3.maxAsyncRequestInflightBytes in conf"; - s3_opt->maxAsyncRequestInflightBytes = 0; - } - if (!conf->GetBoolValue("s3.enableTelemetry", &s3_opt->enableTelemetry)) { - LOG(WARNING) << "Not found s3.enableTelemetry in conf,default to false"; - s3_opt->enableTelemetry = false; - } + &request_option->use_virtual_addressing())); + LOG_IF(FATAL, !conf->GetStringValue("s3.region", &request_option->region())); } struct AwsGetObjectAsyncContext; diff --git a/src/dataaccess/aws/client/aws_crt_s3_client.cpp b/src/dataaccess/aws/client/aws_crt_s3_client.cpp index 515f1a8b9..f90479f63 100644 --- a/src/dataaccess/aws/client/aws_crt_s3_client.cpp +++ b/src/dataaccess/aws/client/aws_crt_s3_client.cpp @@ -32,35 +32,38 @@ #include -#include "dataaccess/aws/aws_s3_common.h" - namespace dingofs { namespace dataaccess { namespace aws { -void AwsCrtS3Client::Init(const S3AdapterOption& option) { +void AwsCrtS3Client::Init(const S3Option& option) { + const auto& global_option = option.global_option(); + const auto& bucket_option = option.bucket_option(); + const auto& request_option = option.request_option(); + CHECK(!initialized_.load()) << "AwsCrtS3Client already initialized"; - LOG(INFO) << "AwsCrtS3Client init ak: " << option.ak << " sk: " << option.sk - << " s3_address: " << option.s3Address - << " bucket_name: " << option.bucketName; + LOG(INFO) << "AwsCrtS3Client init ak: " << bucket_option.ak() + << " sk: " << bucket_option.sk() + << " s3_address: " << bucket_option.endpoint() + << " bucket_name: " << bucket_option.bucket_name(); option_ = option; { auto config = std::make_unique(); // config->scheme = Aws::Http::Scheme(option.scheme); - config->verifySSL = option.verifySsl; - config->region = option.region; - config->maxConnections = option.maxConnections; - config->connectTimeoutMs = option.connectTimeout; - config->requestTimeoutMs = option.requestTimeout; - config->endpointOverride = option.s3Address; - config->useVirtualAddressing = option.useVirtualAddressing; + config->verifySSL = request_option.verify_ssl(); + config->region = request_option.region(); + config->maxConnections = request_option.max_connections(); + config->connectTimeoutMs = request_option.connect_timeout_ms(); + config->requestTimeoutMs = request_option.request_timeout_ms(); + config->endpointOverride = bucket_option.endpoint(); + config->useVirtualAddressing = request_option.use_virtual_addressing(); // TODO : to support // config.throughputTargetGbps = throughput_target_gbps; - if (option.enableTelemetry) { + if (global_option.telemetry_enable()) { LOG(INFO) << "Enable telemetry for aws s3 adapter"; ::opentelemetry::exporter::otlp::OtlpHttpExporterOptions opts; auto span_exporter = @@ -81,7 +84,7 @@ void AwsCrtS3Client::Init(const S3AdapterOption& option) { } client_ = std::make_unique( - Aws::Auth::AWSCredentials(option_.ak, option_.sk), *cfg_, + Aws::Auth::AWSCredentials(bucket_option.ak(), bucket_option.sk()), *cfg_, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never); initialized_.store(true); diff --git a/src/dataaccess/aws/client/aws_crt_s3_client.h b/src/dataaccess/aws/client/aws_crt_s3_client.h index 222ea030e..8e33d73df 100644 --- a/src/dataaccess/aws/client/aws_crt_s3_client.h +++ b/src/dataaccess/aws/client/aws_crt_s3_client.h @@ -24,7 +24,6 @@ #include #include -#include "dataaccess/aws/aws_s3_common.h" #include "dataaccess/aws/client/aws_s3_client.h" namespace dingofs { @@ -37,21 +36,21 @@ class AwsCrtS3Client : public AwsS3Client { ~AwsCrtS3Client() override = default; - void Init(const S3AdapterOption& option) override; + void Init(const S3Option& option) override; std::string GetAk() override { DCHECK(initialized_.load(std::memory_order_relaxed)); - return option_.ak; + return option_.bucket_option().ak(); } std::string GetSk() override { DCHECK(initialized_.load(std::memory_order_relaxed)); - return option_.sk; + return option_.bucket_option().sk(); } std::string GetEndpoint() override { DCHECK(initialized_.load(std::memory_order_relaxed)); - return option_.s3Address; + return option_.bucket_option().endpoint(); } bool BucketExist(std::string bucket) override; @@ -82,7 +81,7 @@ class AwsCrtS3Client : public AwsS3Client { private: std::atomic initialized_{false}; - S3AdapterOption option_; + S3Option option_; std::unique_ptr cfg_{nullptr}; std::unique_ptr client_{nullptr}; diff --git a/src/dataaccess/aws/client/aws_legacy_s3_client.cpp b/src/dataaccess/aws/client/aws_legacy_s3_client.cpp index 4a30d4e04..0f650b666 100644 --- a/src/dataaccess/aws/client/aws_legacy_s3_client.cpp +++ b/src/dataaccess/aws/client/aws_legacy_s3_client.cpp @@ -56,11 +56,16 @@ namespace dingofs { namespace dataaccess { namespace aws { -void AwsLegacyS3Client::Init(const S3AdapterOption& option) { +void AwsLegacyS3Client::Init(const S3Option& option) { + const auto& global_option = option.global_option(); + const auto& bucket_option = option.bucket_option(); + const auto& request_option = option.request_option(); + CHECK(!initialized_.load()) << "AwsLegacyS3Client already initialized"; - LOG(INFO) << "AwsLegacyS3Client init ak: " << option.ak - << " sk: " << option.sk << " s3_address: " << option.s3Address - << " bucket_name: " << option.bucketName; + LOG(INFO) << "AwsLegacyS3Client init ak: " << bucket_option.ak() + << " sk: " << bucket_option.sk() + << " s3_address: " << bucket_option.endpoint() + << " bucket_name: " << bucket_option.bucket_name(); option_ = option; @@ -68,23 +73,23 @@ void AwsLegacyS3Client::Init(const S3AdapterOption& option) { // init config auto config = std::make_unique(); // config->scheme = Aws::Http::Scheme(option.scheme); - config->verifySSL = option.verifySsl; + config->verifySSL = request_option.verify_ssl(); config->userAgent = "S3 Browser"; - config->region = option.region; - config->maxConnections = option.maxConnections; - config->connectTimeoutMs = option.connectTimeout; - config->requestTimeoutMs = option.requestTimeout; - config->endpointOverride = option.s3Address; + config->region = request_option.region(); + config->maxConnections = request_option.max_connections(); + config->connectTimeoutMs = request_option.connect_timeout_ms(); + config->requestTimeoutMs = request_option.request_timeout_ms(); + config->endpointOverride = bucket_option.endpoint(); - if (option.use_thread_pool) { + if (request_option.use_thread_pool()) { LOG(INFO) << "AwsLegacyS3Client init async thread pool thread num = " - << option.asyncThreadNum; + << request_option.async_thread_num(); config->executor = Aws::MakeShared( - "AwsLegacyS3Client", option.asyncThreadNum); + "AwsLegacyS3Client", request_option.async_thread_num()); } - if (option.enableTelemetry) { + if (global_option.telemetry_enable()) { LOG(INFO) << "Enable telemetry for aws s3 adapter"; ::opentelemetry::exporter::otlp::OtlpHttpExporterOptions opts; auto span_exporter = @@ -106,9 +111,9 @@ void AwsLegacyS3Client::Init(const S3AdapterOption& option) { } client_ = std::make_unique( - Aws::Auth::AWSCredentials(option_.ak, option_.sk), *cfg_, + Aws::Auth::AWSCredentials(bucket_option.ak(), bucket_option.sk()), *cfg_, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, - option.useVirtualAddressing); + request_option.use_virtual_addressing()); initialized_.store(true); } diff --git a/src/dataaccess/aws/client/aws_legacy_s3_client.h b/src/dataaccess/aws/client/aws_legacy_s3_client.h index 77b5cd14a..69c41a1c6 100644 --- a/src/dataaccess/aws/client/aws_legacy_s3_client.h +++ b/src/dataaccess/aws/client/aws_legacy_s3_client.h @@ -23,7 +23,6 @@ #include #include -#include "dataaccess/aws/aws_s3_common.h" #include "dataaccess/aws/client/aws_s3_client.h" namespace dingofs { @@ -36,21 +35,21 @@ class AwsLegacyS3Client : public AwsS3Client { ~AwsLegacyS3Client() override = default; - void Init(const S3AdapterOption& option) override; + void Init(const S3Option& option) override; std::string GetAk() override { DCHECK(initialized_.load(std::memory_order_relaxed)); - return option_.ak; + return option_.bucket_option().ak(); } std::string GetSk() override { DCHECK(initialized_.load(std::memory_order_relaxed)); - return option_.sk; + return option_.bucket_option().sk(); } std::string GetEndpoint() override { DCHECK(initialized_.load(std::memory_order_relaxed)); - return option_.s3Address; + return option_.bucket_option().endpoint(); } bool BucketExist(std::string bucket) override; @@ -81,7 +80,7 @@ class AwsLegacyS3Client : public AwsS3Client { private: std::atomic initialized_{false}; - S3AdapterOption option_; + S3Option option_; std::unique_ptr cfg_{nullptr}; std::unique_ptr client_{nullptr}; diff --git a/src/dataaccess/aws/client/aws_s3_client.h b/src/dataaccess/aws/client/aws_s3_client.h index eec0ca1a4..ff8f55ba2 100644 --- a/src/dataaccess/aws/client/aws_s3_client.h +++ b/src/dataaccess/aws/client/aws_s3_client.h @@ -20,6 +20,7 @@ #include #include "dataaccess/aws/aws_s3_common.h" +#include "options/client/s3.h" namespace dingofs { namespace dataaccess { @@ -31,7 +32,7 @@ class AwsS3Client { virtual ~AwsS3Client() = default; - virtual void Init(const S3AdapterOption& option) = 0; + virtual void Init(const S3Option& option) = 0; virtual std::string GetAk() = 0; virtual std::string GetSk() = 0; @@ -43,16 +44,18 @@ class AwsS3Client { const char* buffer, size_t buffer_size) = 0; virtual void PutObjectAsync( - std::string bucket, std::shared_ptr context) = 0; + std::string bucket, + std::shared_ptr context) = 0; virtual int GetObject(std::string bucket, const std::string& key, std::string* data) = 0; virtual int RangeObject(std::string bucket, const std::string& key, char* buf, - off_t offset, size_t len) = 0; + off_t offset, size_t len) = 0; virtual void GetObjectAsync( - std::string bucket, std::shared_ptr context) = 0; + std::string bucket, + std::shared_ptr context) = 0; virtual int DeleteObject(std::string bucket, const std::string& key) = 0; diff --git a/src/dataaccess/aws/s3_adapter.cpp b/src/dataaccess/aws/s3_adapter.cpp index c1f427f0b..f05c7e99f 100644 --- a/src/dataaccess/aws/s3_adapter.cpp +++ b/src/dataaccess/aws/s3_adapter.cpp @@ -31,9 +31,11 @@ #include #include +#include "dataaccess/aws/aws_s3_common.h" #include "dataaccess/aws/client/aws_crt_s3_client.h" #include "dataaccess/aws/client/aws_legacy_s3_client.h" #include "dataaccess/aws/s3_access_log.h" +#include "options/client/s3.h" #include "utils/dingo_define.h" #include "utils/macros.h" @@ -54,19 +56,25 @@ static std::once_flag s3_init_flag; static std::once_flag s3_shutdown_flag; static Aws::SDKOptions aws_sdk_options; -void S3Adapter::Init(const S3AdapterOption& option) { +void S3Adapter::Init(const S3Option& option) { + const auto& global_option = option.global_option(); + const auto& bucket_option = option.bucket_option(); + const auto& request_option = option.request_option(); + const auto& throttle_option = option.throttle_option(); + // TODO: refact this auto init_sdk = [&]() { aws_sdk_options.loggingOptions.logLevel = - Aws::Utils::Logging::LogLevel(option.loglevel); - aws_sdk_options.loggingOptions.defaultLogPrefix = option.logPrefix.c_str(); + Aws::Utils::Logging::LogLevel(global_option.log_level()); + aws_sdk_options.loggingOptions.defaultLogPrefix = + global_option.log_prefix().c_str(); Aws::InitAPI(aws_sdk_options); }; std::call_once(s3_init_flag, init_sdk); - bucket_ = option.bucketName; + bucket_ = bucket_option.bucket_name(); - if (option.use_crt_client) { + if (request_option.use_crt_client()) { s3_client_ = std::make_unique(); } else { // init aws s3 client @@ -77,21 +85,21 @@ void S3Adapter::Init(const S3AdapterOption& option) { { utils::ReadWriteThrottleParams params; - params.iopsTotal.limit = option.iopsTotalLimit; - params.iopsRead.limit = option.iopsReadLimit; - params.iopsWrite.limit = option.iopsWriteLimit; - params.bpsTotal.limit = option.bpsTotalMB * kMB; - params.bpsRead.limit = option.bpsReadMB * kMB; - params.bpsWrite.limit = option.bpsWriteMB * kMB; + params.iopsTotal.limit = throttle_option.iops_total_limit(); + params.iopsRead.limit = throttle_option.iops_read_limit(); + params.iopsWrite.limit = throttle_option.iops_write_limit(); + params.bpsTotal.limit = throttle_option.bps_total_mb(); + params.bpsRead.limit = throttle_option.bps_read_mb(); + params.bpsWrite.limit = throttle_option.bps_write_mb(); throttle_ = std::make_unique(); throttle_->UpdateThrottleParams(params); inflightBytesThrottle_ = std::make_unique( - option.maxAsyncRequestInflightBytes == 0 + request_option.max_async_request_inflight_bytes() == 0 ? UINT64_MAX - : option.maxAsyncRequestInflightBytes); + : request_option.max_async_request_inflight_bytes()); } } @@ -101,7 +109,7 @@ void S3Adapter::Shutdown() { std::call_once(s3_shutdown_flag, shutdown_sdk); } -void S3Adapter::Reinit(const S3AdapterOption& option) { Init(option); } +void S3Adapter::Reinit(const S3Option& option) { Init(option); } std::string S3Adapter::GetS3Ak() { return s3_client_->GetAk(); } diff --git a/src/dataaccess/aws/s3_adapter.h b/src/dataaccess/aws/s3_adapter.h index b848d6884..3110fdc19 100644 --- a/src/dataaccess/aws/s3_adapter.h +++ b/src/dataaccess/aws/s3_adapter.h @@ -33,6 +33,7 @@ #include "dataaccess/accesser_common.h" #include "dataaccess/aws/aws_s3_common.h" #include "dataaccess/aws/client/aws_s3_client.h" +#include "options/client/s3.h" #include "utils/configuration.h" #include "utils/throttle.h" @@ -47,11 +48,11 @@ class S3Adapter { virtual ~S3Adapter() = default; // 初始化S3Adapter - virtual void Init(const S3AdapterOption& option); + virtual void Init(const S3Option& option); static void Shutdown(); - virtual void Reinit(const S3AdapterOption& option); + virtual void Reinit(const S3Option& option); virtual std::string GetS3Ak(); virtual std::string GetS3Sk(); diff --git a/src/dataaccess/s3_accesser.h b/src/dataaccess/s3_accesser.h index be104ec1b..28ac8f76e 100644 --- a/src/dataaccess/s3_accesser.h +++ b/src/dataaccess/s3_accesser.h @@ -19,35 +19,13 @@ #include "dataaccess/accesser.h" #include "dataaccess/aws/s3_adapter.h" +#include "options/client/s3.h" namespace dingofs { namespace dataaccess { using ::dingofs::client::Status; - -struct S3Option { - std::string ak; - std::string sk; - std::string s3Address; - std::string bucketName; - std::string region; - int loglevel; - std::string logPrefix; - bool verifySsl; - int maxConnections; - int connectTimeout; - int requestTimeout; - int asyncThreadNum; - uint64_t maxAsyncRequestInflightBytes; - uint64_t iopsTotalLimit; - uint64_t iopsReadLimit; - uint64_t iopsWriteLimit; - uint64_t bpsTotalMB; - uint64_t bpsReadMB; - uint64_t bpsWriteMB; - bool useVirtualAddressing; - bool enableTelemetry; -}; +using options::client::S3Option; class S3Accesser; using S3AccesserPtr = std::shared_ptr; @@ -57,10 +35,10 @@ using S3AccesserPtr = std::shared_ptr; // use aws-sdk-cpp implement class S3Accesser : public DataAccesser { public: - S3Accesser(const aws::S3AdapterOption& option) : option_(option) {} + S3Accesser(const S3Option& option) : option_(option) {} ~S3Accesser() override = default; - static S3AccesserPtr New(const aws::S3AdapterOption& option) { + static S3AccesserPtr New(const S3Option& option) { return std::make_shared(option); } @@ -83,7 +61,7 @@ class S3Accesser : public DataAccesser { private: static Aws::String S3Key(const std::string& key); - const aws::S3AdapterOption option_; + const S3Option option_; std::unique_ptr client_; }; diff --git a/src/mds/CMakeLists.txt b/src/mds/CMakeLists.txt index be5c7f68e..cf87ca6ab 100644 --- a/src/mds/CMakeLists.txt +++ b/src/mds/CMakeLists.txt @@ -23,7 +23,7 @@ add_subdirectory(schedule) add_subdirectory(heartbeat) add_subdirectory(metric) -add_library(mds_lib +add_library(mds_lib fs_info_wrapper.cpp fs_manager.cpp fs_storage.cpp @@ -45,6 +45,7 @@ target_link_libraries(mds_lib mds_kvstorage_client dingofs_base_lib dingofs_common + dingofs_options dynamic_vlog dingofs_utils aws_s3_adapter diff --git a/src/mds/fs_manager.cpp b/src/mds/fs_manager.cpp index 26d3d7364..6c7a68ba4 100644 --- a/src/mds/fs_manager.cpp +++ b/src/mds/fs_manager.cpp @@ -294,12 +294,11 @@ FSStatusCode FsManager::CreateFs(const pb::mds::CreateFsRequest* request, if (!skip_create_new_fs && detail.has_s3info()) { const auto& s3_info = detail.s3info(); - dataaccess::aws::S3AdapterOption s3_adapter_option = - option_.s3AdapterOption; - s3_adapter_option.ak = s3_info.ak(); - s3_adapter_option.sk = s3_info.sk(); - s3_adapter_option.s3Address = s3_info.endpoint(); - s3_adapter_option.bucketName = s3_info.bucketname(); + options::client::S3Option s3_adapter_option = option_.s3_option; + s3_adapter_option.bucket_option().ak() = s3_info.ak(); + s3_adapter_option.bucket_option().sk() = s3_info.sk(); + s3_adapter_option.bucket_option().endpoint() = s3_info.endpoint(); + s3_adapter_option.bucket_option().bucket_name() = s3_info.bucketname(); auto s3_adapter = std::make_shared(); s3_adapter->Init(s3_adapter_option); diff --git a/src/mds/fs_manager.h b/src/mds/fs_manager.h index 4471238c8..165e8faac 100644 --- a/src/mds/fs_manager.h +++ b/src/mds/fs_manager.h @@ -38,6 +38,7 @@ #include "mds/fs_storage.h" #include "mds/metaserverclient/metaserver_client.h" #include "mds/topology/topology_manager.h" +#include "options/client/s3.h" #include "utils/concurrent/concurrent.h" #include "utils/interruptible_sleeper.h" @@ -48,7 +49,7 @@ struct FsManagerOption { uint32_t backEndThreadRunInterSec; uint32_t spaceReloadConcurrency = 10; uint32_t clientTimeoutSec = 20; - dataaccess::aws::S3AdapterOption s3AdapterOption; + options::client::S3Option s3_option; }; class FsManager { diff --git a/src/mds/mds.cpp b/src/mds/mds.cpp index 8495a1408..b7feb10bc 100644 --- a/src/mds/mds.cpp +++ b/src/mds/mds.cpp @@ -174,7 +174,7 @@ void MDS::InitFsManagerOptions(FsManagerOption* fs_manager_option) { << fs_manager_option->spaceReloadConcurrency; dataaccess::aws::InitS3AdaptorOptionExceptS3InfoOption( - conf_.get(), &fs_manager_option->s3AdapterOption); + conf_.get(), &fs_manager_option->s3_option); } void MDS::Init() { diff --git a/src/mdsv2/background/gc.cc b/src/mdsv2/background/gc.cc index 16521105e..8d5dd3176 100644 --- a/src/mdsv2/background/gc.cc +++ b/src/mdsv2/background/gc.cc @@ -85,7 +85,7 @@ Status CleanDeletedSliceTask::CleanDeletedSlice(const std::string& key, void CleanDeletedFileTask::Run() {} bool GcProcessor::Init() { - dataaccess::aws::S3AdapterOption option; + options::client::S3Option option; data_accessor_ = dataaccess::S3Accesser::New(option); CHECK(data_accessor_->Init()) << "init data accesser fail."; diff --git a/src/metaserver/metaserver.cpp b/src/metaserver/metaserver.cpp index 7a4e2373a..0e292e6ac 100644 --- a/src/metaserver/metaserver.cpp +++ b/src/metaserver/metaserver.cpp @@ -213,8 +213,7 @@ void InitMetaCacheOption(const std::shared_ptr& conf, namespace { std::shared_ptr CreateS3Adaptor( - const dataaccess::aws::S3AdapterOption& o1, - const S3ClientAdaptorOption& o2) { + const options::client::S3Option& o1, const S3ClientAdaptorOption& o2) { // s3_client auto* s3_client = new S3ClientImpl; s3_client->SetAdaptor(std::make_shared()); @@ -243,7 +242,7 @@ void Metaserver::Init() { S3ClientAdaptorOption s3_client_adaptor_option; InitS3Option(conf_, &s3_client_adaptor_option); - dataaccess::aws::S3AdapterOption s3_adapter_option; + options::client::S3Option s3_adapter_option; dataaccess::aws::InitS3AdaptorOptionExceptS3InfoOption(conf_.get(), &s3_adapter_option); diff --git a/src/metaserver/s3/metaserver_s3.cpp b/src/metaserver/s3/metaserver_s3.cpp index 6d608cc0e..dd4038306 100644 --- a/src/metaserver/s3/metaserver_s3.cpp +++ b/src/metaserver/s3/metaserver_s3.cpp @@ -30,7 +30,7 @@ void S3ClientImpl::SetAdaptor( s3Adapter_ = s3Adapter; } -void S3ClientImpl::Init(const dataaccess::aws::S3AdapterOption& option) { +void S3ClientImpl::Init(const options::client::S3Option& option) { s3Adapter_->Init(option); option_ = option; } @@ -38,10 +38,10 @@ void S3ClientImpl::Init(const dataaccess::aws::S3AdapterOption& option) { void S3ClientImpl::Reinit(const std::string& ak, const std::string& sk, const std::string& endpoint, const std::string& bucketName) { - option_.ak = ak; - option_.sk = sk; - option_.s3Address = endpoint; - option_.bucketName = bucketName; + option_.bucket_option().ak() = ak; + option_.bucket_option().sk() = sk; + option_.bucket_option().endpoint() = endpoint; + option_.bucket_option().bucket_name() = bucketName; s3Adapter_->Reinit(option_); } diff --git a/src/metaserver/s3/metaserver_s3.h b/src/metaserver/s3/metaserver_s3.h index 39e66729d..f4626a307 100644 --- a/src/metaserver/s3/metaserver_s3.h +++ b/src/metaserver/s3/metaserver_s3.h @@ -35,7 +35,7 @@ class S3Client { public: S3Client() = default; virtual ~S3Client() = default; - virtual void Init(const dataaccess::aws::S3AdapterOption& option) = 0; + virtual void Init(const options::client::S3Option& option) = 0; virtual int Delete(const std::string& name) = 0; virtual int DeleteBatch(const std::list& nameList) = 0; virtual void Reinit(const std::string& ak, const std::string& sk, @@ -49,7 +49,7 @@ class S3ClientImpl : public S3Client { ~S3ClientImpl() override = default; void SetAdaptor(std::shared_ptr s3Adapter); - void Init(const dataaccess::aws::S3AdapterOption& option) override; + void Init(const options::client::S3Option& option) override; void Reinit(const std::string& ak, const std::string& sk, const std::string& endpoint, const std::string& bucketName) override; @@ -69,7 +69,7 @@ class S3ClientImpl : public S3Client { private: std::shared_ptr s3Adapter_; - dataaccess::aws::S3AdapterOption option_; + options::client::S3Option option_; }; } // namespace metaserver diff --git a/src/metaserver/s3compact_inode.cpp b/src/metaserver/s3compact_inode.cpp index 52daedfa9..7136550c6 100644 --- a/src/metaserver/s3compact_inode.cpp +++ b/src/metaserver/s3compact_inode.cpp @@ -423,10 +423,10 @@ S3Adapter* CompactInodeJob::SetupS3Adapter(uint64_t fsId, s3adapter->GetS3Sk() != s3info.sk() || s3adapter->GetS3Endpoint() != s3info.endpoint()) { auto option = opts_->s3adapterManager->GetBasicS3AdapterOption(); - option.ak = s3info.ak(); - option.sk = s3info.sk(); - option.s3Address = s3info.endpoint(); - option.bucketName = s3info.bucketname(); + option.bucket_option().ak() = s3info.ak(); + option.bucket_option().sk() = s3info.sk(); + option.bucket_option().endpoint() = s3info.endpoint(); + option.bucket_option().bucket_name() = s3info.bucketname(); s3adapter->Reinit(option); } Aws::String bucketName(s3info.bucketname().c_str(), diff --git a/src/metaserver/s3compact_manager.cpp b/src/metaserver/s3compact_manager.cpp index 4bac38e92..61569051c 100644 --- a/src/metaserver/s3compact_manager.cpp +++ b/src/metaserver/s3compact_manager.cpp @@ -29,6 +29,7 @@ #include "absl/memory/memory.h" #include "metaserver/partition.h" #include "metaserver/s3compact_worker.h" +#include "options/client/s3.h" #include "utils/string_util.h" namespace dingofs { @@ -36,7 +37,7 @@ namespace metaserver { using dataaccess::aws::InitS3AdaptorOptionExceptS3InfoOption; using dataaccess::aws::S3Adapter; -using dataaccess::aws::S3AdapterOption; +using options::client::S3Option; using pb::common::S3Info; using utils::Configuration; using utils::InterruptibleSleeper; @@ -89,7 +90,7 @@ void S3AdapterManager::ReleaseS3Adapter(uint64_t index) { used_[index] = false; } -S3AdapterOption S3AdapterManager::GetBasicS3AdapterOption() { return opts_; } +S3Option S3AdapterManager::GetBasicS3AdapterOption() { return opts_; } void S3CompactWorkQueueOption::Init(std::shared_ptr conf) { std::string mdsAddrsStr; @@ -98,10 +99,10 @@ void S3CompactWorkQueueOption::Init(std::shared_ptr conf) { conf->GetValueFatalIfFail("global.ip", &metaserverIpStr); conf->GetValueFatalIfFail("global.port", &metaserverPort); // leave ak,sk,addr,bucket,chunksize,blocksize blank - s3opts.ak = ""; - s3opts.sk = ""; - s3opts.s3Address = ""; - s3opts.bucketName = ""; + s3opts.bucket_option().set_ak(""); + s3opts.bucket_option().set_sk(""); + s3opts.bucket_option().set_endpoint(""); + s3opts.bucket_option().set_bucket_name(""); InitS3AdaptorOptionExceptS3InfoOption(conf.get(), &s3opts); conf->GetValueFatalIfFail("s3compactwq.enable", &enable); conf->GetValueFatalIfFail("s3compactwq.thread_num", &threadNum); diff --git a/src/metaserver/s3compact_manager.h b/src/metaserver/s3compact_manager.h index b72f9511f..f2889ed5c 100644 --- a/src/metaserver/s3compact_manager.h +++ b/src/metaserver/s3compact_manager.h @@ -32,6 +32,7 @@ #include "metaserver/s3compact.h" #include "metaserver/s3compact_worker.h" #include "metaserver/s3infocache.h" +#include "options/client/s3.h" #include "utils/configuration.h" namespace dingofs { @@ -44,22 +45,23 @@ class S3AdapterManager { uint64_t size_; // same size as worker thread count std::vector> s3adapters_; std::vector used_; - dataaccess::aws::S3AdapterOption opts_; + options::client::S3Option opts_; public: explicit S3AdapterManager(uint64_t size, - const dataaccess::aws::S3AdapterOption& opts) + const options::client::S3Option& opts) : inited_(false), size_(size), opts_(opts) {} virtual ~S3AdapterManager() = default; virtual void Init(); virtual void Deinit(); virtual std::pair GetS3Adapter(); virtual void ReleaseS3Adapter(uint64_t index); - virtual dataaccess::aws::S3AdapterOption GetBasicS3AdapterOption(); + virtual options::client::S3Option GetBasicS3AdapterOption(); }; struct S3CompactWorkQueueOption { - dataaccess::aws::S3AdapterOption s3opts; + options::client::S3Option s3opts; + bool enable; uint64_t threadNum; uint64_t fragmentThreshold; diff --git a/src/client/vfs_old/kvclient/CMakeLists.txt b/src/options/CMakeLists.txt similarity index 58% rename from src/client/vfs_old/kvclient/CMakeLists.txt rename to src/options/CMakeLists.txt index ddf3dba7c..209e76a60 100644 --- a/src/client/vfs_old/kvclient/CMakeLists.txt +++ b/src/options/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2024 dingodb.com, Inc. All Rights Reserved +# Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,21 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# Define the BASE_FLAGS and DINGO_DEFAULT_COPTS variables -add_library(client_memcached_client - kvclient_manager.cpp - memcache_client.cpp +file(GLOB DINGOFS_OPTIONS_LIB_SRCS + "options.cpp" + "cache/*.cpp" + "client/*.cpp" ) -target_link_libraries(client_memcached_client - PROTO_OBJS - stub_metric - dingofs_utils - aws_s3_adapter - absl::strings +add_library(dingofs_options ${DINGOFS_OPTIONS_LIB_SRCS}) + +target_link_libraries(dingofs_options + toml11::toml11 + gflags::gflags glog::glog - brpc::brpc - ${LIBMEMCACHED_LIBRARY} - ${hashkit_LIBRARIES} -) \ No newline at end of file +) diff --git a/src/options/README.md b/src/options/README.md new file mode 100644 index 000000000..e3a422a58 --- /dev/null +++ b/src/options/README.md @@ -0,0 +1,101 @@ +DingoFS options module +=== + +Basic Usage +--- + +main.toml: + +```toml +title = "person description" + +[person] +name = "Jack" +age = 18 +``` + +main.cpp: + +```cpp +#include "dingofs/options/options.h" + +class PersonOption : public BaseOption { + BIND_string(name, "none", "person name"); + BIND_int32(age, 0, "person age"); +}; + +class GlobalOption : public BaseOption { + BIND_string(title, "none", "title"); + BIND_suboption(person_option, "person", PersonOption); +}; + +int main() { + GlobalOption option; + if (!option.Parse("main.toml")) { + return -1; + } + + std::cout << "title: " << option.title() << std::endl; + std::cout << "person.name: " << option.person_option().name() << std::endl; + std::cout << "person.age: " << option.person_option().age() << std::endl; + + return 0; +} +``` + +Support On-Fly +--- + +main.toml: + +```toml +rpc_timeout_s = 3 +``` + +main.cpp: + +```cpp +#include "dingofs/options/options.h" + +DEFINE_ONFLY_int32(dingofs_meta_rpc_timeout_s, 1, "meta rpc timeout (seconds)") + +class GlobalOption : public BaseOption { + BIND_ONFLY_int32(rpc_timeout_s, dingofs_meta_rpc_timeout_s); +}; + +int main() { + GlobalOption option; + std::cout << "rpc_timeout_s: " << option.rpc_timeout_s() << std::endl; // 1 + + if (!option.Parse("main.toml")) { + return -1; + } + std::cout << "rpc_timeout_s: " << option.rpc_timeout_s() << std::endl; // 3 + + FLAGS_dingofs_meta_rpc_timeout_s = 10; + std::cout << "rpc_timeout_s: " << option.rpc_timeout_s() << std::endl; // 10 + + return 0; +} +``` + +Global Option +--- + +use option like gflags. + +main.toml: + +``` +name = "client" +``` + +main.cpp: + +```cpp +DECLARE_OPTION(app_client, AppOption); +DEFINE_OPTION(app_client, AppOption); +if (OPTIONS_app_client.Parse("main.toml")) { + std::cout << OPTIONS_app_client.name() << std::endl; +} +``` diff --git a/src/options/cache/app.cpp b/src/options/cache/app.cpp new file mode 100644 index 000000000..ae00667dd --- /dev/null +++ b/src/options/cache/app.cpp @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * Project: DingoFS + * Created Date: 2025-05-08 + * Author: Jingli Chen (Wine93) + */ + +#include "options/cache/app.h" + +#include "options/options.h" + +namespace dingofs { +namespace options { +namespace cache { + +DEFINE_OPTION(cache, AppOption); + +} // namespace cache +} // namespace options +} // namespace dingofs diff --git a/src/options/cache/app.h b/src/options/cache/app.h new file mode 100644 index 000000000..94139c54e --- /dev/null +++ b/src/options/cache/app.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: DingoFS + * Created Date: 2025-05-07 + * Author: Jingli Chen (Wine93) + */ + +#ifndef DINGOFS_SRC_OPTIONS_CACHE_APP_H_ +#define DINGOFS_SRC_OPTIONS_CACHE_APP_H_ + +#include "options/cache/blockcache.h" +#include "options/cache/cachegroup.h" +#include "options/cache/global.h" + +// options level: +// app -> global -> cachegroup -> blockcache + +namespace dingofs { +namespace options { +namespace cache { + +class AppOption : public BaseOption { + BIND_suboption(global_option, "global", GlobalOption); + BIND_suboption(cache_group_option, "cache_group", CacheGroupOption); + BIND_suboption(block_cache_option, "block_cache", BlockCacheOption); +}; + +DECLARE_OPTION(cache, AppOption); + +} // namespace cache +} // namespace options +} // namespace dingofs + +#endif // DINGOFS_SRC_OPTIONS_CACHE_APP_H_ diff --git a/src/options/cache/blockcache.cpp b/src/options/cache/blockcache.cpp new file mode 100644 index 000000000..4cd43c046 --- /dev/null +++ b/src/options/cache/blockcache.cpp @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: DingoFS + * Created Date: 2025-05-07 + * Author: Jingli Chen (Wine93) + */ + +#include "options/cache/blockcache.h" + +namespace dingofs { +namespace options { +namespace cache { + +DEFINE_ONFLY_bool(block_cache_logging, true, ""); +DEFINE_ONFLY_bool(block_cache_drop_page_cache, true, + "drop stage block page cache"); + +} // namespace cache +} // namespace options +} // namespace dingofs diff --git a/src/options/cache/blockcache.h b/src/options/cache/blockcache.h new file mode 100644 index 000000000..d809d71ab --- /dev/null +++ b/src/options/cache/blockcache.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: DingoFS + * Created Date: 2025-05-07 + * Author: Jingli Chen (Wine93) + */ + +#ifndef DINGOFS_SRC_OPTIONS_CACHE_BLOCKCACHE_H_ +#define DINGOFS_SRC_OPTIONS_CACHE_BLOCKCACHE_H_ + +#include "options/options.h" + +namespace dingofs { +namespace options { +namespace cache { + +class DiskStateOption : public BaseOption { + BIND_uint32(tick_duration_s, 60, ""); + BIND_uint32(normal2unstable_io_error_num, 3, ""); + BIND_uint32(unstable2normal_io_succ_num, 10, ""); + BIND_uint32(unstable2down_s, 1800, ""); + BIND_uint32(disk_check_duration_ms, 3000, ""); +}; + +DECLARE_ONFLY_bool(block_cache_logging); +DECLARE_ONFLY_bool(block_cache_drop_page_cache); + +class DiskCacheOption : public BaseOption { + BIND_string_array(cache_dir, STR_ARRAY{"/tmp/dingofs-cache"}, ""); + BIND_int32(cache_size_mb, 10240, ""); + BIND_double(free_space_ratio, 0.1, ""); + BIND_int32(cache_expire_s, 259200, ""); + BIND_int32(cleanup_expire_interval_ms, 1000, ""); + BIND_ONFLY_bool(drop_page_cache, block_cache_drop_page_cache); + BIND_int32(ioring_iodepth, 128, ""); + BIND_int32(ioring_blksize, 1048576, ""); + BIND_bool(ioring_prefetch, true, ""); + BIND_suboption(disk_state_option, "disk_state", DiskStateOption); +}; + +class BlockCacheOption : public BaseOption { + BIND_ONFLY_bool(logging, block_cache_logging); + BIND_string(cache_store, "disk", ""); + BIND_bool(stage, true, ""); + BIND_bool(stage_bandwidth_throttle_enable, false, ""); + BIND_int32(stage_bandwidth_throttle_mb, 10240, ""); + BIND_int32(upload_stage_workers, 10, ""); + BIND_int32(upload_stage_queue_size, 10000, ""); + BIND_int32(prefetch_workers, 128, ""); + BIND_int32(prefetch_queue_size, 128, ""); + BIND_suboption(disk_cache_option, "disk_cache", DiskCacheOption); +}; + +} // namespace cache +} // namespace options +} // namespace dingofs + +#endif // DINGOFS_SRC_OPTIONS_CACHE_BLOCKCACHE_H_ diff --git a/src/options/cache/cachegroup.h b/src/options/cache/cachegroup.h new file mode 100644 index 000000000..01e8264c6 --- /dev/null +++ b/src/options/cache/cachegroup.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: DingoFS + * Created Date: 2025-05-07 + * Author: Jingli Chen (Wine93) + */ + +#ifndef DINGOFS_SRC_OPTIONS_CACHE_CACHEGROUP_H_ +#define DINGOFS_SRC_OPTIONS_CACHE_CACHEGROUP_H_ + +#include "options/options.h" + +namespace dingofs { +namespace options { +namespace cache { + +class CacheGroupOption : public BaseOption { + BIND_string(group_name, "default", ""); + BIND_string(listen_ip, "127.0.0.1", ""); + BIND_uint32(listen_port, 19301, ""); + BIND_uint32(group_weight, 100, ""); + BIND_uint32(max_range_size_kb, 256, ""); + BIND_string(metadata_filepath, "/var/log/cache_group_meta", ""); + BIND_uint32(load_members_interval_ms, 1000, ""); +}; + +} // namespace cache +} // namespace options +} // namespace dingofs + +#endif // DINGOFS_SRC_OPTIONS_CACHE_CACHEGROUP_H_ diff --git a/src/options/cache/global.h b/src/options/cache/global.h new file mode 100644 index 000000000..cf628fed5 --- /dev/null +++ b/src/options/cache/global.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: DingoFS + * Created Date: 2025-05-08 + * Author: Jingli Chen (Wine93) + */ + +#ifndef DINGOFS_SRC_OPTIONS_CACHE_GLOBAL_H_ +#define DINGOFS_SRC_OPTIONS_CACHE_GLOBAL_H_ + +#include "options/options.h" + +namespace dingofs { +namespace options { +namespace cache { + +class GlobalOption : public BaseOption {}; + +} // namespace cache +} // namespace options +} // namespace dingofs + +#endif // DINGOFS_SRC_OPTIONS_CACHE_GLOBAL_H_ diff --git a/src/options/client/app.cpp b/src/options/client/app.cpp new file mode 100644 index 000000000..8860bcf95 --- /dev/null +++ b/src/options/client/app.cpp @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * Project: DingoFS + * Created Date: 2025-05-08 + * Author: Jingli Chen (Wine93) + */ + +#include "options/client/app.h" + +namespace dingofs { +namespace options { +namespace client { + +DEFINE_OPTION(client, AppOption); + +} // namespace client +} // namespace options +} // namespace dingofs diff --git a/src/options/client/app.h b/src/options/client/app.h new file mode 100644 index 000000000..d73dd5700 --- /dev/null +++ b/src/options/client/app.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: DingoFS + * Created Date: 2025-05-07 + * Author: Jingli Chen (Wine93) + */ + +#ifndef DINGOFS_SRC_OPTIONS_CLIENT_APP_H_ +#define DINGOFS_SRC_OPTIONS_CLIENT_APP_H_ + +#include "options/cache/blockcache.h" +#include "options/client/data.h" +#include "options/client/fuse.h" +#include "options/client/global.h" +#include "options/client/meta.h" +#include "options/client/rpc.h" +#include "options/client/s3.h" +#include "options/client/vfs.h" +#include "options/options.h" + +// options level layout: +// ┌─> meta -> rpc +// app -> global -> fuse -> vfs -| +// └─> data -> { blockcache, s3 } +namespace dingofs { +namespace options { +namespace client { + +class AppOption : public BaseOption { + BIND_suboption(global_option, "global", GlobalOption); + BIND_suboption(fuse_option, "fuse", FuseOption); + BIND_suboption(vfs_option, "vfs", VFSOption); + BIND_suboption(meta_option, "meta", MetaOption); + BIND_suboption(data_option, "data", DataOption); + BIND_suboption(rpc_option, "rpc", RPCOption); + BIND_suboption(block_cache_option, "block_cache", cache::BlockCacheOption); + BIND_suboption(s3_option, "s3", S3Option); +}; + +DECLARE_OPTION(client, AppOption); + +} // namespace client +} // namespace options +} // namespace dingofs + +#endif // DINGOFS_SRC_OPTIONS_CLIENT_APP_H_ diff --git a/src/options/client/data.h b/src/options/client/data.h new file mode 100644 index 000000000..461845693 --- /dev/null +++ b/src/options/client/data.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: DingoFS + * Created Date: 2025-05-07 + * Author: Jingli Chen (Wine93) + */ + +#ifndef DINGOFS_SRC_OPTIONS_CLIENT_DATA_H_ +#define DINGOFS_SRC_OPTIONS_CLIENT_DATA_H_ + +#include "options/options.h" + +namespace dingofs { +namespace options { +namespace client { + +class BackgroundFlushOption : public BaseOption { + BIND_int32(interval_ms, 1000, ""); + BIND_double(trigger_force_memory_ratio, 0.90, ""); +}; + +class FlushFileOption : public BaseOption { + BIND_int32(flush_workers, 10, ""); + BIND_int32(flush_queue_size, 500, ""); +}; + +class FlushChunkOption : public BaseOption { + BIND_int32(flush_workers, 10, ""); + BIND_int32(flush_queue_size, 500, ""); +}; + +class FlushSliceOption : public BaseOption { + BIND_int32(flush_workers, 10, ""); + BIND_int32(flush_queue_size, 500, ""); + BIND_int32(stay_in_memory_max_s, 5, ""); +}; + +class PageOption : public BaseOption { + BIND_int32(page_size, 65536, ""); + BIND_int32(total_size_mb, 1024, ""); + BIND_bool(use_pool, true, ""); +}; + +class DataOption : public BaseOption { + BIND_suboption(background_flush_option, "background_flush", + BackgroundFlushOption); + BIND_suboption(flush_file_option, "flush_file", FlushFileOption); + BIND_suboption(flush_chunk_option, "flush_chunk", FlushChunkOption); + BIND_suboption(flush_slice_option, "flush_slice", FlushSliceOption); + BIND_suboption(page_option, "page", PageOption); +}; + +} // namespace client +} // namespace options +} // namespace dingofs + +#endif // DINGOFS_SRC_OPTIONS_CLIENT_DATA_H_ diff --git a/src/options/client/fuse.h b/src/options/client/fuse.h new file mode 100644 index 000000000..6440685c2 --- /dev/null +++ b/src/options/client/fuse.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: DingoFS + * Created Date: 2025-05-07 + * Author: Jingli Chen (Wine93) + */ + +#ifndef DINGOFS_SRC_OPTIONS_CLIENT_FUSE_H_ +#define DINGOFS_SRC_OPTIONS_CLIENT_FUSE_H_ + +#include "options/options.h" + +namespace dingofs { +namespace options { +namespace client { + +class FuseConnInfoOption : public BaseOption { + BIND_bool(want_splice_move, false, ""); + BIND_bool(want_splice_read, false, ""); + BIND_bool(want_splice_write, false, ""); + BIND_bool(want_auto_inval_data, true, ""); +}; + +class FuseFileInfoOption : public BaseOption { + BIND_bool(direct_io, false, ""); + BIND_bool(keep_cache, true, ""); +}; + +class FuseOption : public BaseOption { + BIND_suboption(conn_info, "conn_info", FuseConnInfoOption); + BIND_suboption(file_info, "file_info", FuseFileInfoOption); +}; + +} // namespace client +} // namespace options +} // namespace dingofs + +#endif // DINGOFS_SRC_OPTIONS_CLIENT_FUSE_H_ diff --git a/src/options/client/global.cpp b/src/options/client/global.cpp new file mode 100644 index 000000000..2131389e8 --- /dev/null +++ b/src/options/client/global.cpp @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * Project: DingoFS + * Created Date: 2025-05-08 + * Author: Jingli Chen (Wine93) + */ + +#include "options/client/global.h" + +#include "options/options.h" + +namespace dingofs { +namespace options { +namespace client {} // namespace client +} // namespace options +} // namespace dingofs diff --git a/src/options/client/global.h b/src/options/client/global.h new file mode 100644 index 000000000..1075f7ca0 --- /dev/null +++ b/src/options/client/global.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: DingoFS + * Created Date: 2025-05-08 + * Author: Jingli Chen (Wine93) + */ + +#ifndef DINGOFS_SRC_OPTIONS_CLIENT_GLOBAL_H_ +#define DINGOFS_SRC_OPTIONS_CLIENT_GLOBAL_H_ + +#include "options/options.h" + +namespace dingofs { +namespace options { +namespace client { + +class GlobalOption : public BaseOption { + BIND_string(log_dir, "/tmp", ""); + BIND_int32(vlog_level, 0, ""); + BIND_bool(access_logging, true, ""); + BIND_uint32(dump_server_start_port, 9000, ""); +}; + +} // namespace client +} // namespace options +} // namespace dingofs + +#endif // DINGOFS_SRC_OPTIONS_CLIENT_GLOBAL_H_ diff --git a/src/options/client/meta.h b/src/options/client/meta.h new file mode 100644 index 000000000..0298eec68 --- /dev/null +++ b/src/options/client/meta.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: DingoFS + * Created Date: 2025-05-07 + * Author: Jingli Chen (Wine93) + */ + +#ifndef DINGOFS_SRC_OPTIONS_CLIENT_META_H_ +#define DINGOFS_SRC_OPTIONS_CLIENT_META_H_ + +#include "options/options.h" + +namespace dingofs { +namespace options { +namespace client { + +namespace v1 { + +class MDSOption {}; + +class MetaOption : public BaseOption { + BIND_suboption(mds_option, "mds", MDSOption); +}; + +}; // namespace v1 + +namespace v2 { + +class MDSOption : public BaseOption { + BIND_uint32(rpc_timeout_ms, 2000, ""); + BIND_uint32(rpc_retry_times, 3, ""); + BIND_uint32(client_send_request_retry_times, 3, ""); +}; + +class MetaOption : public BaseOption { + BIND_suboption(mds_option, "mds", MDSOption); +}; + +}; // namespace v2 + +class MetaOption : public BaseOption { + BIND_suboption(v1, "v1", v1::MetaOption); + BIND_suboption(v2, "v2", v2::MetaOption); +}; + +} // namespace client +} // namespace options +} // namespace dingofs + +#endif // DINGOFS_SRC_OPTIONS_CLIENT_META_H_ diff --git a/src/options/client/rpc.h b/src/options/client/rpc.h new file mode 100644 index 000000000..8c3142f68 --- /dev/null +++ b/src/options/client/rpc.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: DingoFS + * Created Date: 2025-05-07 + * Author: Jingli Chen (Wine93) + */ + +#ifndef DINGOFS_SRC_OPTIONS_CLIENT_RPC_H_ +#define DINGOFS_SRC_OPTIONS_CLIENT_RPC_H_ + +#include "options/options.h" + +namespace dingofs { +namespace options { +namespace client { + +class MdsRpcOption : public BaseOption { + BIND_int32(max_rpc_timeout_ms, 2000, ""); + BIND_int32(rpc_timeout_ms, 500, ""); + BIND_int32(rpc_retry_interval_us, 50000, ""); + BIND_int32(max_failed_times_before_change_addr, 2, ""); + BIND_int32(normal_retry_times_before_trigger_wait, 3, ""); + BIND_int32(wait_sleep_ms, 1000, ""); + BIND_string(addrs, "127.0.0.1:6700,127.0.0.1:6701,127.0.0.1:6702", ""); +}; + +class MetaRpcOption : public BaseOption { + BIND_int32(rpc_timeout_ms, 1000, ""); + BIND_int32(rpc_stream_idle_timeout_ms, 500, ""); + + BIND_int32(max_retires, 10, ""); + BIND_int32(max_internal_retries, 3, ""); + BIND_int32(retry_interval_us, 100000, ""); + BIND_int32(list_dentry_limit, 65536, ""); + + BIND_int32(max_rpc_timeout_ms, 8000, ""); + BIND_int32(max_retry_sleep_interval_us, 8000000, ""); + BIND_int32(min_retry_times_force_timeout_backoff, 5, ""); + BIND_int32(max_retry_times_before_consider_suspend, 20, ""); + BIND_int32(batch_inode_attr_limit, 10000, ""); +}; + +class RPCOption : public BaseOption {}; + +} // namespace client +} // namespace options +} // namespace dingofs + +#endif // DINGOFS_SRC_OPTIONS_CLIENT_RPC_H_ diff --git a/src/options/client/s3.h b/src/options/client/s3.h new file mode 100644 index 000000000..59de6bac5 --- /dev/null +++ b/src/options/client/s3.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: DingoFS + * Created Date: 2025-05-07 + * Author: Jingli Chen (Wine93) + */ + +#ifndef DINGOFS_SRC_OPTIONS_CLIENT_S3_H_ +#define DINGOFS_SRC_OPTIONS_CLIENT_S3_H_ + +#include "options/options.h" + +namespace dingofs { +namespace options { +namespace client { + +class S3GlobalOption : public BaseOption { + BIND_int32(log_level, 4, ""); + BIND_string(log_prefix, "/tmp/", ""); + BIND_bool(telemetry_enable, false, ""); +}; + +class S3BucketOption : public BaseOption { + BIND_string(bucket_name, "", ""); + BIND_string(ak, "", ""); + BIND_string(sk, "", ""); + BIND_string(endpoint, "", ""); + BIND_int32(block_size, 4194304, ""); + BIND_int32(chunk_size, 67108864, ""); + BIND_string(object_prefix, "", ""); +}; + +class S3RequestOption : public BaseOption { + BIND_string(region, "us-east-1", ""); + BIND_bool(use_virtual_addressing, false, ""); + BIND_bool(verify_ssl, false, ""); + BIND_int32(max_connections, 32, ""); + BIND_int32(connect_timeout_ms, 60000, ""); + BIND_int32(request_timeout_ms, 10000, ""); + BIND_bool(use_crt_client, false, ""); + BIND_bool(use_thread_pool, true, "only work when use_crt_client is false"); + BIND_int32(async_thread_num, true, "only work when use_crt_client is false"); + BIND_uint64(max_async_request_inflight_bytes, 0, ""); +}; + +class S3ThrottleOption : public BaseOption { + BIND_uint64(iops_total_limit, 0, ""); + BIND_uint64(iops_read_limit, 0, ""); + BIND_uint64(iops_write_limit, 0, ""); + BIND_uint64(bps_total_mb, 0, ""); + BIND_uint64(bps_read_mb, 0, ""); + BIND_uint64(bps_write_mb, 0, ""); +}; + +class S3Option : public BaseOption { + BIND_suboption(global_option, "global", S3GlobalOption); + BIND_suboption(bucket_option, "bucket", S3BucketOption); + BIND_suboption(request_option, "request", S3RequestOption); + BIND_suboption(throttle_option, "throttle", S3ThrottleOption); +}; + +// do nothing + +} // namespace client +} // namespace options +} // namespace dingofs + +#endif // DINGOFS_SRC_OPTIONS_CLIENT_S3_H_ diff --git a/src/options/client/vfs.cpp b/src/options/client/vfs.cpp new file mode 100644 index 000000000..e160986a9 --- /dev/null +++ b/src/options/client/vfs.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * Project: DingoFS + * Created Date: 2025-05-08 + * Author: Jingli Chen (Wine93) + */ + +#include "options/client/app.h" + +namespace dingofs { +namespace options { +namespace client {} // namespace client +} // namespace options +} // namespace dingofs diff --git a/src/options/client/vfs.h b/src/options/client/vfs.h new file mode 100644 index 000000000..d36cf998d --- /dev/null +++ b/src/options/client/vfs.h @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: DingoFS + * Created Date: 2025-05-07 + * Author: Jingli Chen (Wine93) + */ + +#ifndef DINGOFS_SRC_OPTIONS_CLIENT_VFS_H_ +#define DINGOFS_SRC_OPTIONS_CLIENT_VFS_H_ + +#include "options/client/data.h" +#include "options/client/meta.h" +#include "options/options.h" + +namespace dingofs { +namespace options { +namespace client { + +class KernelCacheOption : public BaseOption { + BIND_double(attr_timeout_s, 30, ""); + BIND_double(dir_attr_timeout_s, 30, ""); + BIND_double(entry_timeout_s, 30, ""); + BIND_double(dir_entry_timeout_s, 30, ""); +}; + +class LookupCacheOption : public BaseOption { + BIND_int32(negative_timeout_s, 0, ""); + BIND_int32(min_uses, 1, ""); + BIND_int32(lru_size, 100000, ""); +}; + +class DirCacheOption : public BaseOption { + BIND_int32(lru_size, 5000000, ""); +}; + +class AttrWatcherOption : public BaseOption { + BIND_int32(lru_size, 5000000, ""); +}; + +class RpcOption : public BaseOption { + BIND_int32(list_dentry_limit, 65536, ""); +}; + +class DeferSyncOption : public BaseOption { + BIND_int32(delay_s, 3, ""); + BIND_int32(defer_dir_mtime, false, ""); +}; + +class QuotaOption : public BaseOption { + BIND_uint32(flush_quota_interval_s, 0, ""); + BIND_uint32(load_quota_interval_s, 0, ""); +}; + +class LeaseOption : public BaseOption { + BIND_uint32(lease_times_us, 20000000, ""); + BIND_uint32(refresh_times_per_lease, 5, ""); +}; + +class ThrottleOption : public BaseOption { + BIND_uint32(avg_write_bytes, 0, ""); + BIND_uint32(burst_write_bytes, 0, ""); + BIND_uint32(burst_write_bytes_s, 180, ""); + BIND_uint32(avg_write_iops, 0, ""); + BIND_uint32(burst_write_iops, 0, ""); + BIND_uint32(burst_write_iops_s, 180, ""); + BIND_uint32(avg_read_bytes, 0, ""); + BIND_uint32(burst_read_bytes, 0, ""); + BIND_uint32(burst_read_bytes_s, 180, ""); + BIND_uint32(avg_read_iops, 0, ""); + BIND_uint32(burst_read_iops, 0, ""); + BIND_uint32(burst_read_iops_s, 180, ""); +}; + +class VFSOption : public BaseOption { + BIND_bool(cto, true, ""); + BIND_string(nocto_suffix, " ", ""); + BIND_int32(max_name_length, 255, ""); + BIND_bool(disable_xattr, true, ""); + + BIND_suboption(kernel_cache_option, "kernel", KernelCacheOption); + BIND_suboption(lookup_cache_option, "lookup_cache", LookupCacheOption); + BIND_suboption(dir_cache_option, "dir_cache", DirCacheOption); + BIND_suboption(attr_watcher_option, "attr_watcher", AttrWatcherOption); + BIND_suboption(rpc_option, "rpc_option", RpcOption); + BIND_suboption(defer_sync_option, "defer_sync", DeferSyncOption); + BIND_suboption(quota_option, "quota", QuotaOption); + BIND_suboption(lease_option, "lease", LeaseOption); + + BIND_suboption(meta_option, "meta", MetaOption); + BIND_suboption(data_option, "data", DataOption); +}; + +} // namespace client +} // namespace options +} // namespace dingofs + +#endif // DINGOFS_SRC_OPTIONS_CLIENT_VFS_H diff --git a/src/options/options.cpp b/src/options/options.cpp new file mode 100644 index 000000000..587e5ba13 --- /dev/null +++ b/src/options/options.cpp @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * Project: DingoFS + * Created Date: 2025-05-06 + * Author: Jingli Chen (Wine93) + */ + +#include "options/options.h" + +#include + +namespace dingofs { +namespace options { + +namespace internal { + +bool pass_bool(const char*, bool) { return true; } +bool pass_int32(const char*, int32_t) { return true; } +bool pass_uint32(const char*, uint32_t) { return true; } +bool pass_int64(const char*, int64_t) { return true; } +bool pass_uint64(const char*, uint64_t) { return true; } +bool pass_double(const char*, double) { return true; } +bool pass_string(const char*, std::string) { return true; } + +} // namespace internal + +bool BaseOption::Parse(const std::string& filepath) { + const auto input = toml::try_parse(filepath); + if (input.is_ok()) { + const auto& root = input.unwrap(); + return Walk(root); + } + return false; +} + +bool BaseOption::Walk(const toml::value& node) { + if (!node.is_table()) { + return true; + } + + for (auto [key, value] : node.as_table()) { + bool succ = + value.is_table() ? HandleTable(key, value) : HandleNormal(key, value); + if (!succ) { + std::cerr << "handle kv pair failed: key = " << key << std::endl; + return false; + } + } + return true; +} + +bool BaseOption::HandleTable(const std::string& key, const toml::value& value) { + auto iter = childs_.find(key); + if (iter == childs_.end()) { + std::cerr << "unknown section: " << key << std::endl; + return false; + } + + const auto& child = iter->second; + return child->Walk(value); +} + +bool BaseOption::HandleNormal(const std::string& key, + const toml::value& value) { + auto iter = items_.find(key); + if (iter == items_.end()) { + std::cerr << "unknown option: " << key << std::endl; + return false; + } + + auto& item = iter->second; + return item->SetValue(value); +} + +} // namespace options +} // namespace dingofs diff --git a/src/options/options.h b/src/options/options.h new file mode 100644 index 000000000..adb1b8ee5 --- /dev/null +++ b/src/options/options.h @@ -0,0 +1,307 @@ +/* + * Copyright (c) 2025 dingodb.com, Inc. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: DingoFS + * Created Date: 2025-05-06 + * Author: Jingli Chen (Wine93) + */ + +#ifndef DINGOFS_SRC_OPTIONS_OPTIONS_H_ +#define DINGOFS_SRC_OPTIONS_OPTIONS_H_ + +#include +#include +#include + +#include +#include + +namespace dingofs { +namespace options { + +// interface for option item +class IItem { + public: + virtual ~IItem() = default; + + virtual bool SetValue(const toml::value& value) = 0; +}; + +template +class Item : public IItem { + public: + Item(const std::string& name, T* value, const std::string& comment) + : name_(name), value_(value), comment_(comment) {} + + bool SetValue(const toml::value& value) override { + if constexpr (std::is_same_v) { + return SetBoolean(value); + } else if constexpr (std::is_integral_v) { + return SetInteger(value); + } else if constexpr (std::is_floating_point_v) { + return SetFloating(value); + } else if constexpr (std::is_same_v) { + return SetString(value); + } else if constexpr (std::is_same_v>) { + return SetArray(value); + } else if constexpr (std::is_same_v>) { + return SetArray(value); + } + return false; + } + + private: + bool SetBoolean(const toml::value& value) { + if (value.is_boolean()) { + *value_ = toml::get(value); + return true; + } + return false; + } + + bool SetInteger(const toml::value& value) { + if (value.is_integer()) { + *value_ = toml::get(value); + return true; + } + return false; + } + + bool SetInt(const toml::value& value) { + if (value.is_integer()) { + *value_ = toml::get(value); + return true; + } + return false; + } + + bool SetFloating(const toml::value& value) { + if (value.is_floating()) { + *value_ = toml::get(value); + return true; + } + return false; + } + + bool SetString(const toml::value& value) { + if (value.is_string()) { + *value_ = toml::get(value); + return true; + } + return false; + } + + bool SetArray(const toml::value& value) { + if (value.is_array()) { + *value_ = toml::get(value); + return true; + } + return false; + } + + std::string name_; + T* value_; + std::string comment_; +}; + +class BaseOption { + public: + BaseOption() = default; + + bool Parse(const std::string& filepath); + + protected: + bool Walk(const toml::value& node); + + bool HandleTable(const std::string& key, const toml::value& value); + + bool HandleNormal(const std::string& key, const toml::value& value); + + std::unordered_map childs_; + std::unordered_map items_; +}; + +namespace internal { + +bool pass_bool(const char*, bool); +bool pass_int32(const char*, int32_t); +bool pass_uint32(const char*, uint32_t); +bool pass_int64(const char*, int64_t); +bool pass_uint64(const char*, uint64_t); +bool pass_double(const char*, double); +bool pass_string(const char*, std::string); + +}; // namespace internal + +} // namespace options +} // namespace dingofs + +// macros + +// bind suboption +#define BIND_suboption(name, child_name, cls) \ + public: \ + cls& name() { return name##_; } \ + const cls& name() const { return name##_; } \ + \ + private: \ + cls name##_; \ + [[maybe_unused]] bool name##Insert_ = [this]() { \ + BaseOption::childs_[child_name] = \ + reinterpret_cast(&this->name##_); \ + return true; \ + }() + +// bind base +#define BIND_base(T, name, default_value, comment) \ + public: \ + T& name() { return name##_; } \ + const T& name() const { return name##_; } \ + void set_##name(T value) { name##Item_.SetValue(value); } \ + \ + private: \ + T name##_{default_value}; \ + Item name##Item_ = Item(#name, &name##_, comment); \ + [[maybe_unused]] bool name##Insert_ = [this]() { \ + BaseOption::items_[#name] = reinterpret_cast(&this->name##Item_); \ + return true; \ + }() + +// bind gflags +#define BIND_ONFLY_base(T, name, gflag_name) \ + public: \ + T& name() { return FLAGS_##gflag_name; } \ + const T& name() const { return FLAGS_##gflag_name; } \ + void set_##name(T value) { FLAGS_##gflag_name = value; } \ + \ + private: \ + Item name##Item_ = Item(#name, &FLAGS_##gflag_name, ""); \ + [[maybe_unused]] bool name##Insert_ = [this]() { \ + BaseOption::items_[#name] = reinterpret_cast(&this->name##Item_); \ + return true; \ + }() + +// bind_* +#define BIND_bool(name, default_value, comment) \ + BIND_base(bool, name, default_value, comment) + +#define BIND_int32(name, default_value, comment) \ + BIND_base(int32_t, name, default_value, comment) + +#define BIND_uint32(name, default_value, comment) \ + BIND_base(uint32_t, name, default_value, comment) + +#define BIND_int64(name, default_value, comment) \ + BIND_base(int64_t, name, default_value, comment) + +#define BIND_uint64(name, default_value, comment) \ + BIND_base(uint64_t, name, default_value, comment) + +#define BIND_double(name, default_value, comment) \ + BIND_base(double, name, default_value, comment) + +#define BIND_string(name, default_value, comment) \ + BIND_base(std::string, name, default_value, comment) + +#define BIND_string_array(name, default_value, comment) \ + BIND_base(std::vector, name, default_value, comment) + +#define BIND_uint32_array(name, default_value, comment) \ + BIND_base(std::vector, name, default_value, comment) + +// declare_onfly_* +#define DECLARE_ONFLY_bool(name) DECLARE_bool(name); +#define DECLARE_ONFLY_int32(name) DECLARE_int32(name); +#define DECLARE_ONFLY_int64(name) DECLARE_int64(name); +#define DECLARE_ONFLY_uint32(name) DECLARE_uint32(name); +#define DECLARE_ONFLY_uint64(name) DECLARE_uint64(name); +#define DECLARE_ONFLY_double(name) DECLARE_double(name); +#define DECLARE_ONFLY_string(name) DECLARE_string(name); + +// declare_onfly_* +#define DEFINE_ONFLY_bool(name, default_value, comment) \ + DEFINE_bool(name, default_value, comment); \ + DEFINE_validator(name, &internal::pass_bool); + +#define DEFINE_ONFLY_int32(name, default_value, comment) \ + DEFINE_int32(name, default_value, comment); \ + DEFINE_validator(name, &internal::pass_int32); + +#define DEFINE_ONFLY_uint32(name, default_value, comment) \ + DEFINE_uint32(name, default_value, comment); \ + DEFINE_validator(name, &internal::pass_uint32); + +#define DEFINE_ONFLY_int64(name, default_value, comment) \ + DEFINE_int64(name, default_value, comment); \ + DEFINE_validator(name, &internal::pass_int64); + +#define DEFINE_ONFLY_uint64(name, default_value, comment) \ + DEFINE_uint64(name, default_value, comment); \ + DEFINE_validator(name, &internal::pass_uint64); + +#define DEFINE_ONFLY_double(name, default_value, comment) \ + DEFINE_double(name, default_value, comment); \ + DEFINE_validator(name, &internal::pass_double); + +#define DEFINE_ONFLY_string(name, default_value, comment) \ + DEFINE_string(name, default_value, comment); \ + DEFINE_validator(name, &internal::pass_string); + +// bind_onfly_* +#define BIND_ONFLY_bool(name, gflag_name) \ + BIND_ONFLY_base(bool, name, gflag_name) + +#define BIND_ONFLY_int32(name, gflag_name) \ + BIND_ONFLY_base(int32_t, name, gflag_name) + +#define BIND_ONFLY_uint32(name, gflag_name) \ + BIND_ONFLY_base(uint32_t, name, gflag_name) + +#define BIND_ONFLY_int64(name, gflag_name) \ + BIND_ONFLY_base(int64_t, name, gflag_name) + +#define BIND_ONFLY_uint64(name, gflag_name) \ + BIND_ONFLY_base(uint64_t, name, gflag_name) + +#define BIND_ONFLY_double(name, gflag_name) \ + BIND_ONFLY_base(double, name, gflag_name) + +#define BIND_ONFLY_string(name, gflag_name) \ + BIND_ONFLY_base(std::string, name, gflag_name) + +// utils +#define STR_ARRAY std::vector +#define UINT32_ARRAY std::vector + +// /* We always want to import declared variables, dll or no */ \ +// namespace fL##shorttype { extern GFLAGS_DLL_DECLARE_FLAG type FLAGS_##name; } \ +// using fL##shorttype::FLAGS_##name + +// options +#define DECLARE_OPTION(name, type) \ + namespace __g_options__ { \ + extern type OPTIONS_##name; \ + } \ + using __g_options__::OPTIONS_##name + +#define DEFINE_OPTION(name, type) \ + namespace __g_options__ { \ + type OPTIONS_##name; \ + } \ + using __g_options__::OPTIONS_##name + +#endif // DINGOFS_SRC_OPTIONS_OPTIONS_H_ diff --git a/test/client/vfs_old/client_memcache_test.cpp b/test/client/vfs_old/client_memcache_test.cpp deleted file mode 100644 index a5a38bbb4..000000000 --- a/test/client/vfs_old/client_memcache_test.cpp +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright (c) 2020 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Project: dingo - * File Created: 2022-09-26 - * Author: fansehep (YangFan) - */ - -#include -#include - -#include -#include -#include -#include - -#include "absl/strings/str_cat.h" -#include "client/vfs_old/kvclient/kvclient_manager.h" -#include "client/vfs_old/kvclient/memcache_client.h" -#include "utils/concurrent/count_down_event.h" - -using dingofs::utils::CountDownEvent; - -namespace dingofs { -namespace client { - -class MemCachedTest : public ::testing::Test { - public: - MemCachedTest() = default; - void SetUp() { - auto hostname = "127.0.0.1"; - auto port = 18080; - memcached_pid = ::fork(); - if (0 > memcached_pid) { - ASSERT_FALSE(true); - } else if (0 == memcached_pid) { - std::string memcached_port = "-p " + std::to_string(port); - ASSERT_EQ(0, execlp("memcached", "memcached", "-uroot", - memcached_port.c_str(), nullptr)); - } - - std::shared_ptr client(new MemCachedClient()); - ASSERT_EQ(true, client->AddServer(hostname, port)); - ASSERT_EQ(true, client->PushServer()); - common::KVClientManagerOpt opt; - opt.setThreadPooln = 2; - opt.getThreadPooln = 2; - ASSERT_EQ(true, manager_.Init(opt, client)); - - // wait memcached server start - std::string errorlog; - int retry = 0; - bool ret = false; - do { - if ((ret = client->Set("1", "2", 1, &errorlog)) || retry > 100) { - break; - } - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - retry++; - } while (1); - ASSERT_TRUE(ret) << absl::StrCat("memcache start failed, errorlog: ", - errorlog); - LOG(INFO) << "=============== memcache start ok"; - } - - void TearDown() { - auto str = absl::StrCat("kill -9 ", memcached_pid); - ::system(str.c_str()); - std::this_thread::sleep_for(std::chrono::seconds(3)); - } - pid_t memcached_pid; - KVClientManager manager_; -}; - -TEST_F(MemCachedTest, MultiThreadTask) { - // prepare data - std::vector workers; - std::vector> kvstr = {{"123", "1231"}, - {"456", "4561"}, - {"789", "7891"}, - {"234", "2341"}, - {"890", "8901"}}; - - // set - CountDownEvent taskEvent(5); - for (int i = 0; i < 5; i++) { - workers.emplace_back([&, i]() { - auto task = std::make_shared( - kvstr[i].first, kvstr[i].second.c_str(), kvstr[i].second.length()); - task->done = [&taskEvent](const std::shared_ptr& task) { - taskEvent.Signal(); - }; - manager_.Set(task); - }); - } - taskEvent.Wait(); - ASSERT_EQ(5, - manager_.GetClientMetricForTesting()->kvClientSet.latency.count()); - - // get - for (int i = 0; i < 5; i++) { - workers.emplace_back([&, i]() { - CountDownEvent taskEvent(1); - char* result = new char[4]; - auto task = - std::make_shared(kvstr[i].first, result, 0, 4); - task->done = [&taskEvent](const std::shared_ptr& task) { - taskEvent.Signal(); - }; - manager_.Get(task); - taskEvent.Wait(); - ASSERT_EQ(0, memcmp(result, kvstr[i].second.c_str(), 4)); - ASSERT_TRUE(task->res); - }); - } - for (auto& iter : workers) { - if (iter.joinable()) { - iter.join(); - } - } -} -} // namespace client -} // namespace dingofs diff --git a/test/metaserver/mock_metaserver_s3.h b/test/metaserver/mock_metaserver_s3.h index 7c83b33ad..61771ad42 100644 --- a/test/metaserver/mock_metaserver_s3.h +++ b/test/metaserver/mock_metaserver_s3.h @@ -41,7 +41,7 @@ class MockS3Client : public S3Client { MockS3Client() = default; ~MockS3Client() override = default; - MOCK_METHOD1(Init, void(const dataaccess::aws::S3AdapterOption& options)); + MOCK_METHOD1(Init, void(const options::client::S3Option& options)); MOCK_METHOD1(Delete, int(const std::string& name)); MOCK_METHOD1(DeleteBatch, int(const std::list& nameList)); MOCK_METHOD4(Reinit, void(const std::string& ak, const std::string& sk, diff --git a/test/metaserver/s3compact/mock_s3_adapter.h b/test/metaserver/s3compact/mock_s3_adapter.h index ee60517d2..269553317 100644 --- a/test/metaserver/s3compact/mock_s3_adapter.h +++ b/test/metaserver/s3compact/mock_s3_adapter.h @@ -41,8 +41,7 @@ namespace metaserver { class MockS3AdapterManager : public S3AdapterManager { public: - MockS3AdapterManager(uint64_t size, - const dataaccess::aws::S3AdapterOption& opts) + MockS3AdapterManager(uint64_t size, const options::client::S3Option& opts) : S3AdapterManager(size, opts) {} ~MockS3AdapterManager() override = default; MOCK_METHOD0(Init, void()); @@ -57,7 +56,7 @@ class MockS3Adapter : public S3Adapter { MOCK_METHOD1(Init, void(const std::string&)); MOCK_METHOD0(Deinit, void()); - MOCK_METHOD1(Reinit, void(const dataaccess::aws::S3AdapterOption& opt)); + MOCK_METHOD1(Reinit, void(const options::client::S3Option& opt)); MOCK_METHOD0(GetS3Ak, std::string()); MOCK_METHOD0(GetS3Sk, std::string()); MOCK_METHOD0(GetS3Endpoint, std::string()); diff --git a/test/metaserver/s3compact/s3compact_test.cpp b/test/metaserver/s3compact/s3compact_test.cpp index ae970366e..e65502b29 100644 --- a/test/metaserver/s3compact/s3compact_test.cpp +++ b/test/metaserver/s3compact/s3compact_test.cpp @@ -72,7 +72,7 @@ class S3CompactTest : public ::testing::Test { opts_.s3ReadMaxRetry = 2; opts_.s3ReadRetryInterval = 1; uint64_t s3adapterSize = 10; - dataaccess::aws::S3AdapterOption opts; + options::client::S3Option opts; s3adapterManager_ = std::make_shared(s3adapterSize, opts); s3adapter_ = std::make_shared(); @@ -179,7 +179,7 @@ TEST_F(S3CompactTest, test_S3InfoCache) { } TEST_F(S3CompactTest, test_S3AdapterManager) { - dataaccess::aws::S3AdapterOption opt; + options::client::S3Option opt; opt.loglevel = 0; opt.s3Address = ""; opt.ak = "";