Skip to content

Commit d06e929

Browse files
author
zackcao
committed
Merge branch 'dev-auto-retry-request-config-util' into 'master' (merge request !61)
[合规重试]添加重试相关配置和工具类
2 parents 783aacc + 38c1383 commit d06e929

File tree

8 files changed

+209
-49
lines changed

8 files changed

+209
-49
lines changed

include/cos_config.h

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,11 @@
1111
#include "util/log_util.h"
1212

1313
namespace qcloud_cos {
14+
15+
#define COS_DEFAULT_MAX_RETRY_TIMES 3
16+
17+
#define COS_DEFAULT_RETRY_INTERVAL_MS 100
18+
1419
class CosConfig {
1520
public:
1621
/// \brief CosConfig构造函数
@@ -31,7 +36,9 @@ class CosConfig {
3136
m_dest_domain(""),
3237
m_is_domain_same_to_host(false),
3338
m_is_domain_same_to_host_enable(false),
34-
m_config_parsed(false) {}
39+
m_config_parsed(false),
40+
m_max_retry_times(COS_DEFAULT_MAX_RETRY_TIMES),
41+
m_retry_interval_ms(COS_DEFAULT_RETRY_INTERVAL_MS) {}
3542

3643
/// \brief CosConfig构造函数
3744
///
@@ -52,7 +59,9 @@ class CosConfig {
5259
m_dest_domain(""),
5360
m_is_domain_same_to_host(false),
5461
m_is_domain_same_to_host_enable(false),
55-
m_config_parsed(false) {}
62+
m_config_parsed(false),
63+
m_max_retry_times(COS_DEFAULT_MAX_RETRY_TIMES),
64+
m_retry_interval_ms(COS_DEFAULT_RETRY_INTERVAL_MS) {}
5665

5766
/// \brief CosConfig构造函数
5867
///
@@ -74,7 +83,9 @@ class CosConfig {
7483
m_dest_domain(""),
7584
m_is_domain_same_to_host(false),
7685
m_is_domain_same_to_host_enable(false),
77-
m_config_parsed(false) {}
86+
m_config_parsed(false),
87+
m_max_retry_times(COS_DEFAULT_MAX_RETRY_TIMES),
88+
m_retry_interval_ms(COS_DEFAULT_RETRY_INTERVAL_MS) {}
7889

7990
/// \brief CosConfig复制构造函数
8091
///
@@ -92,6 +103,8 @@ class CosConfig {
92103
m_is_domain_same_to_host = config.m_is_domain_same_to_host;
93104
m_is_domain_same_to_host_enable = config.m_is_domain_same_to_host;
94105
m_config_parsed = config.m_config_parsed;
106+
m_max_retry_times = config.m_max_retry_times;
107+
m_retry_interval_ms = config.m_retry_interval_ms;
95108
}
96109

97110
/// \brief CosConfig赋值构造函数
@@ -110,6 +123,8 @@ class CosConfig {
110123
m_is_domain_same_to_host = config.m_is_domain_same_to_host;
111124
m_is_domain_same_to_host_enable = config.m_is_domain_same_to_host;
112125
m_config_parsed = config.m_config_parsed;
126+
m_max_retry_times = config.m_max_retry_times;
127+
m_retry_interval_ms = config.m_retry_interval_ms;
113128
return *this;
114129
}
115130

@@ -198,6 +213,14 @@ class CosConfig {
198213
/// \brief 设置日志回调
199214
void SetLogCallback(const LogCallback log_callback);
200215

216+
uint64_t GetMaxRetryTimes() const;
217+
218+
void SetMaxRetryTimes(uint64_t max_retry_times);
219+
220+
uint64_t GetRetryIntervalMs() const;
221+
222+
void SetRetryIntervalMs(uint64_t retry_interval_ms);
223+
201224
static bool JsonObjectGetStringValue(
202225
const Poco::JSON::Object::Ptr& json_object, const std::string& key,
203226
std::string* value);
@@ -222,6 +245,8 @@ class CosConfig {
222245
bool m_is_domain_same_to_host;
223246
bool m_is_domain_same_to_host_enable;
224247
bool m_config_parsed;
248+
uint64_t m_max_retry_times;
249+
uint64_t m_retry_interval_ms;
225250
};
226251

227252
typedef std::shared_ptr<CosConfig> SharedConfig;

include/util/base_op_util.h

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#ifndef COS_CPP_SDK_V5_INCLUDE_UTIL_BASE_OP_UTIL_H_
2+
#define COS_CPP_SDK_V5_INCLUDE_UTIL_BASE_OP_UTIL_H_
3+
4+
#include <utility>
5+
6+
#include "cos_config.h"
7+
#include "op/cos_result.h"
8+
9+
namespace qcloud_cos {
10+
class BaseOpUtil {
11+
public:
12+
explicit BaseOpUtil(SharedConfig cos_conf) : m_config(std::move(cos_conf)) {}
13+
14+
BaseOpUtil() = default;
15+
16+
bool ShouldChangeBackupDomain(const CosResult &result, const uint32_t &request_num, bool is_ci_req = false) const;
17+
18+
void SleepBeforeRetry(const uint32_t &request_num) const;
19+
20+
std::string GetRealUrl(const std::string& host, const std::string& path, bool is_https, bool is_generate_presigned_url = false) const;
21+
22+
uint64_t GetMaxRetryTimes() const;
23+
24+
static std::string ChangeHostSuffix(const std::string& host);
25+
26+
private:
27+
SharedConfig m_config;
28+
29+
bool UseDefaultDomain() const;
30+
};
31+
} // namespace qcloud_cos
32+
#endif // COS_CPP_SDK_V5_INCLUDE_UTIL_BASE_OP_UTIL_H_

include/util/http_sender.h

Lines changed: 6 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -36,22 +36,6 @@ class HttpSender {
3636
SSLCtxCallback ssl_ctx_cb = nullptr,
3737
void *user_data = nullptr);
3838

39-
static int SendRequest(const SharedTransferHandler& handler,
40-
const std::string& http_method,
41-
const std::string& url_str,
42-
const std::map<std::string, std::string>& req_params,
43-
const std::map<std::string, std::string>& req_headers,
44-
const std::string& req_body,
45-
uint64_t conn_timeout_in_ms,
46-
uint64_t recv_timeout_in_ms,
47-
std::map<std::string, std::string>* resp_headers,
48-
std::ostream& resp_stream, std::string* err_msg,
49-
bool is_check_md5 = false,
50-
bool is_verify_cert = true,
51-
const std::string& ca_location = "",
52-
SSLCtxCallback ssl_ctx_cb = nullptr,
53-
void *user_data = nullptr);
54-
5539
static int SendRequest(const SharedTransferHandler& handler,
5640
const std::string& http_method,
5741
const std::string& url_str,
@@ -72,16 +56,19 @@ class HttpSender {
7256
const std::string& url_str,
7357
const std::map<std::string, std::string>& req_params,
7458
const std::map<std::string, std::string>& req_headers,
75-
std::istream& is, uint64_t conn_timeout_in_ms,
59+
std::istream& is,
60+
uint64_t conn_timeout_in_ms,
7661
uint64_t recv_timeout_in_ms,
7762
std::map<std::string, std::string>* resp_headers,
78-
std::ostream& resp_stream, std::string* err_msg,
63+
std::ostream& resp_stream,
64+
std::string* err_msg,
7965
bool is_check_md5 = false,
8066
bool is_verify_cert = true,
8167
const std::string& ca_location = "",
8268
SSLCtxCallback ssl_ctx_cb = nullptr,
8369
void *user_data = nullptr,
84-
const char *req_body_buf = nullptr, size_t req_body_len = 0);
70+
const char *req_body_buf = nullptr,
71+
size_t req_body_len = 0);
8572

8673
static int SendRequest(const SharedTransferHandler& handler,
8774
const std::string& http_method,

src/cos_config.cpp

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,9 @@ CosConfig::CosConfig(const std::string& config_file)
2121
m_intranet_addr(""),
2222
m_dest_domain(""),
2323
m_is_domain_same_to_host(false),
24-
m_config_parsed(false) {
24+
m_config_parsed(false),
25+
m_max_retry_times(COS_DEFAULT_MAX_RETRY_TIMES),
26+
m_retry_interval_ms(COS_DEFAULT_RETRY_INTERVAL_MS) {
2527
if (InitConf(config_file)) {
2628
m_config_parsed = true;
2729
}
@@ -114,6 +116,10 @@ bool CosConfig::InitConf(const std::string& config_file) {
114116
JsonObjectGetStringValue(object, "Region", &m_region);
115117
m_region = StringUtil::Trim(m_region);
116118

119+
JsonObjectGetIntegerValue(object, "RetryIntervalMs", &m_retry_interval_ms);
120+
121+
JsonObjectGetIntegerValue(object, "MaxRetryTimes", &m_max_retry_times);
122+
117123
uint64_t integer_value;
118124

119125
//设置签名超时时间,单位:秒
@@ -231,6 +237,22 @@ std::string CosConfig::GetTmpToken() const {
231237
return token;
232238
}
233239

240+
uint64_t CosConfig::GetMaxRetryTimes() const {
241+
return m_max_retry_times;
242+
}
243+
244+
void CosConfig::SetMaxRetryTimes(uint64_t max_retry_count) {
245+
m_max_retry_times = max_retry_count;
246+
}
247+
248+
uint64_t CosConfig::GetRetryIntervalMs() const {
249+
return m_retry_interval_ms;
250+
}
251+
252+
void CosConfig::SetRetryIntervalMs(uint64_t retry_interval_ms) {
253+
m_retry_interval_ms = retry_interval_ms;
254+
}
255+
234256
void CosConfig::SetConfigCredentail(const std::string& access_key,
235257
const std::string& secret_key,
236258
const std::string& tmp_token) {

src/op/base_op.cpp

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,7 @@
2020
#include "trsf/transfer_handler.h"
2121

2222
namespace qcloud_cos {
23-
24-
SimpleDnsCache& GetGlobalDnsCacheInstance() {
25-
static SimpleDnsCache dns_cache(CosSysConfig::GetDnsCacheSize(),
26-
CosSysConfig::GetDnsCacheExpireSeconds());
27-
return dns_cache;
28-
}
23+
SimpleDnsCache& GetGlobalDnsCacheInstance();
2924

3025
CosConfig BaseOp::GetCosConfig() const { return *m_config; }
3126

src/util/base_op_util.cpp

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
2+
#include "util/base_op_util.h"
3+
#include "cos_sys_config.h"
4+
#include <thread>
5+
#include "util/simple_dns_cache.h"
6+
#include "util/codec_util.h"
7+
8+
namespace qcloud_cos {
9+
SimpleDnsCache& GetGlobalDnsCacheInstance() {
10+
static SimpleDnsCache dns_cache(CosSysConfig::GetDnsCacheSize(),
11+
CosSysConfig::GetDnsCacheExpireSeconds());
12+
return dns_cache;
13+
}
14+
15+
bool BaseOpUtil::ShouldChangeBackupDomain(const CosResult &result, const uint32_t &request_num, const bool is_ci_req) const
16+
{
17+
if (is_ci_req) {
18+
// 请求到万象的, 不切域名
19+
return false;
20+
}
21+
if (!CosSysConfig::GetRetryChangeDomain()) {
22+
// 没开启开关, 不切域名
23+
return false;
24+
}
25+
if (!UseDefaultDomain()) {
26+
// 未使用默认域名, 不切域名
27+
return false;
28+
}
29+
if (!result.GetXCosRequestId().empty()) {
30+
// 响应中有x-cos-request-id, 说明请求到了 COS, 不切域名
31+
return false;
32+
}
33+
const int statusCode = result.GetHttpStatus();
34+
if (statusCode == 301 || statusCode == 302 || statusCode == 307) {
35+
// 3xx 的响应码只要满足切换条件就切换域名, 不需要等最后一次重试
36+
return true;
37+
}
38+
if (request_num + 1 < m_config->GetMaxRetryTimes()) {
39+
// 还没到最大重试次数, 不切域名
40+
return false;
41+
}
42+
return statusCode >= 500;
43+
}
44+
45+
bool BaseOpUtil::UseDefaultDomain() const {
46+
if (m_config && m_config->GetSetIntranetOnce() && m_config->IsUseIntranet() && !m_config->GetIntranetAddr().empty()
47+
|| CosSysConfig::IsUseIntranet() && !CosSysConfig::GetIntranetAddr().empty()
48+
|| (!m_config->GetDestDomain().empty())
49+
|| !CosSysConfig::GetDestDomain().empty()
50+
|| m_config->GetRegion() == "accelerate") {
51+
return false;
52+
}
53+
return true;
54+
}
55+
56+
std::string BaseOpUtil::GetRealUrl(const std::string& host, const std::string& path,
57+
bool is_https, bool is_generate_presigned_url) const
58+
{
59+
// 1. host优先级,私有ip > 自定义域名 > DNS cache > 默认域名
60+
std::string dest_uri;
61+
std::string dest_host = host;
62+
std::string dest_path = path;
63+
std::string dest_protocal = "http://"; // NOCA:HttpHardcoded(ignore)
64+
if (is_https) {
65+
dest_protocal = "https://";
66+
}
67+
68+
if (dest_path.empty() || '/' != dest_path[0]) {
69+
dest_path = "/" + dest_path;
70+
}
71+
72+
if (m_config &&
73+
m_config->GetSetIntranetOnce() &&
74+
m_config->IsUseIntranet() &&
75+
!m_config->GetIntranetAddr().empty() && !is_generate_presigned_url) {
76+
dest_host = m_config->GetIntranetAddr();
77+
} else if (CosSysConfig::IsUseIntranet() &&
78+
!CosSysConfig::GetIntranetAddr().empty() && !is_generate_presigned_url) {
79+
dest_host = CosSysConfig::GetIntranetAddr();
80+
} else if (m_config &&
81+
(!m_config->GetDestDomain().empty())) {
82+
dest_host = m_config->GetDestDomain();
83+
} else if (!CosSysConfig::GetDestDomain().empty()) {
84+
dest_host = CosSysConfig::GetDestDomain();
85+
} else if (CosSysConfig::GetUseDnsCache() && !is_generate_presigned_url) {
86+
dest_host = GetGlobalDnsCacheInstance().Resolve(host);
87+
}
88+
89+
dest_uri = dest_protocal + dest_host + CodecUtil::EncodeKey(dest_path);
90+
SDK_LOG_DBG("dest_uri: %s", dest_uri.c_str());
91+
return dest_uri;
92+
}
93+
94+
uint64_t BaseOpUtil::GetMaxRetryTimes() const
95+
{
96+
return m_config->GetMaxRetryTimes();
97+
}
98+
99+
void BaseOpUtil::SleepBeforeRetry(const uint32_t& request_num) const
100+
{
101+
const uint32_t interval_ms = m_config->GetRetryIntervalMs() * (request_num + 1);
102+
std::this_thread::sleep_for(std::chrono::milliseconds(interval_ms));
103+
}
104+
105+
std::string BaseOpUtil::ChangeHostSuffix(const std::string& host) {
106+
const std::string old_suffix = ".myqcloud.com";
107+
const std::string new_suffix = ".tencentcos.cn";
108+
109+
const size_t suffix_pos = host.rfind(old_suffix);
110+
if (suffix_pos != std::string::npos) {
111+
std::string new_host = host.substr(0, suffix_pos) + new_suffix;
112+
return new_host;
113+
}
114+
return host;
115+
}
116+
} // namespace qcloud_cos

src/util/http_sender.cpp

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -50,25 +50,6 @@ int HttpSender::SendRequest(
5050
return ret;
5151
}
5252

53-
int HttpSender::SendRequest(
54-
const SharedTransferHandler& handler, const std::string& http_method,
55-
const std::string& url_str,
56-
const std::map<std::string, std::string>& req_params,
57-
const std::map<std::string, std::string>& req_headers,
58-
const std::string& req_body, uint64_t conn_timeout_in_ms,
59-
uint64_t recv_timeout_in_ms,
60-
std::map<std::string, std::string>* resp_headers, std::ostream& resp_stream,
61-
std::string* err_msg, bool is_check_md5,
62-
bool is_verify_cert, const std::string& ca_location,
63-
SSLCtxCallback ssl_ctx_cb, void *user_data) {
64-
std::istringstream is(req_body);
65-
int ret = SendRequest(handler, http_method, url_str, req_params, req_headers,
66-
is, conn_timeout_in_ms, recv_timeout_in_ms,
67-
resp_headers, resp_stream, err_msg, is_check_md5,
68-
is_verify_cert, ca_location, ssl_ctx_cb, user_data);
69-
return ret;
70-
}
71-
7253
int HttpSender::SendRequest(
7354
const SharedTransferHandler& handler, const std::string& http_method,
7455
const std::string& url_str,

unittest/conf/config.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,7 @@
1717
"IsDomainSameToHost":false,
1818
"DestDomain":"",
1919
"IsUseIntranet":false,
20-
"IntranetAddr":""
20+
"IntranetAddr":"",
21+
"RetryIntervalMs":100,
22+
"MaxRetryTimes": 3
2123
}

0 commit comments

Comments
 (0)