Skip to content

Commit 620fe3d

Browse files
agin719agin719
authored
Merge pull request #186 from tencentyun/feature_zackcao_8a0d868b
add retry policy
2 parents 18e2b98 + e6a6f8f commit 620fe3d

25 files changed

+1454
-886
lines changed

‎include/cos_config.h‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,11 @@
1111
#include "util/log_util.h"
1212

1313
namespace qcloud_cos {
14+
15+
#define COS_DEFAULT_MAX_RETRY_TIMES 3
16+
17+
#define COS_DEFAULT_RETRY_INTERVAL_MS 100
18+
1419
class CosConfig {
1520
public:
1621
/// \brief CosConfig构造函数
@@ -31,7 +36,9 @@ class CosConfig {
3136
m_dest_domain(""),
3237
m_is_domain_same_to_host(false),
3338
m_is_domain_same_to_host_enable(false),
34-
m_config_parsed(false) {}
39+
m_config_parsed(false),
40+
m_max_retry_times(COS_DEFAULT_MAX_RETRY_TIMES),
41+
m_retry_interval_ms(COS_DEFAULT_RETRY_INTERVAL_MS) {}
3542

3643
/// \brief CosConfig构造函数
3744
///
@@ -52,7 +59,9 @@ class CosConfig {
5259
m_dest_domain(""),
5360
m_is_domain_same_to_host(false),
5461
m_is_domain_same_to_host_enable(false),
55-
m_config_parsed(false) {}
62+
m_config_parsed(false),
63+
m_max_retry_times(COS_DEFAULT_MAX_RETRY_TIMES),
64+
m_retry_interval_ms(COS_DEFAULT_RETRY_INTERVAL_MS) {}
5665

5766
/// \brief CosConfig构造函数
5867
///
@@ -74,7 +83,9 @@ class CosConfig {
7483
m_dest_domain(""),
7584
m_is_domain_same_to_host(false),
7685
m_is_domain_same_to_host_enable(false),
77-
m_config_parsed(false) {}
86+
m_config_parsed(false),
87+
m_max_retry_times(COS_DEFAULT_MAX_RETRY_TIMES),
88+
m_retry_interval_ms(COS_DEFAULT_RETRY_INTERVAL_MS) {}
7889

7990
/// \brief CosConfig复制构造函数
8091
///
@@ -92,6 +103,8 @@ class CosConfig {
92103
m_is_domain_same_to_host = config.m_is_domain_same_to_host;
93104
m_is_domain_same_to_host_enable = config.m_is_domain_same_to_host;
94105
m_config_parsed = config.m_config_parsed;
106+
m_max_retry_times = config.m_max_retry_times;
107+
m_retry_interval_ms = config.m_retry_interval_ms;
95108
}
96109

97110
/// \brief CosConfig赋值构造函数
@@ -110,6 +123,8 @@ class CosConfig {
110123
m_is_domain_same_to_host = config.m_is_domain_same_to_host;
111124
m_is_domain_same_to_host_enable = config.m_is_domain_same_to_host;
112125
m_config_parsed = config.m_config_parsed;
126+
m_max_retry_times = config.m_max_retry_times;
127+
m_retry_interval_ms = config.m_retry_interval_ms;
113128
return *this;
114129
}
115130

@@ -198,6 +213,14 @@ class CosConfig {
198213
/// \brief 设置日志回调
199214
void SetLogCallback(const LogCallback log_callback);
200215

216+
uint64_t GetMaxRetryTimes() const;
217+
218+
void SetMaxRetryTimes(uint64_t max_retry_times);
219+
220+
uint64_t GetRetryIntervalMs() const;
221+
222+
void SetRetryIntervalMs(uint64_t retry_interval_ms);
223+
201224
static bool JsonObjectGetStringValue(
202225
const Poco::JSON::Object::Ptr& json_object, const std::string& key,
203226
std::string* value);
@@ -222,6 +245,8 @@ class CosConfig {
222245
bool m_is_domain_same_to_host;
223246
bool m_is_domain_same_to_host_enable;
224247
bool m_config_parsed;
248+
uint64_t m_max_retry_times;
249+
uint64_t m_retry_interval_ms;
225250
};
226251

227252
typedef std::shared_ptr<CosConfig> SharedConfig;

‎include/cos_params.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ const char kReqHeaderDate[] = "Date";
5151
const char kReqHeaderServer[] = "Server";
5252
const char kReqHeaderXCosReqId[] = "x-cos-request-id";
5353
const char kReqHeaderXCosTraceId[] = "x-cos-trace-id";
54+
const char kReqHeaderXCosSdkRetry[] = "x-cos-sdk-retry";
5455

5556
// Response Header
5657
const char kRespHeaderLastModified[] = "Last-Modified";

‎include/op/base_op.h‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
#ifndef COS_CPP_SDK_V5_INCLUDE_OP_BASE_OP_H_
33
#define COS_CPP_SDK_V5_INCLUDE_OP_BASE_OP_H_
44

5-
#include <inttypes.h>
65
#include <stdint.h>
76

87
#include <map>
@@ -11,6 +10,7 @@
1110
#include "cos_config.h"
1211
#include "op/cos_result.h"
1312
#include "trsf/transfer_handler.h"
13+
#include "util/base_op_util.h"
1414

1515
namespace qcloud_cos {
1616

@@ -22,7 +22,7 @@ class BaseOp {
2222
/// \brief BaseOp构造函数
2323
///
2424
/// \param cos_conf Cos配置
25-
explicit BaseOp(const SharedConfig& cos_conf) : m_config(cos_conf) {}
25+
explicit BaseOp(const SharedConfig& cos_conf) : m_config(cos_conf), m_op_util(m_config) {}
2626

2727
BaseOp() {}
2828

@@ -51,12 +51,8 @@ class BaseOp {
5151

5252
bool IsDomainSameToHost() const;
5353

54-
bool UseDefaultDomain() const;
55-
5654
bool IsDefaultHost(const std::string &host) const;
5755

58-
std::string ChangeHostSuffix(const std::string &host);
59-
6056
/// \brief 封装了cos Service/Bucket/Object 相关接口的通用操作,
6157
/// 包括签名计算、请求发送、返回内容解析等
6258
///
@@ -128,6 +124,28 @@ class BaseOp {
128124
protected:
129125
bool CheckConfigValidation() const;
130126
SharedConfig m_config;
127+
BaseOpUtil m_op_util;
128+
129+
private:
130+
CosResult NormalRequest(
131+
const std::string& host, const std::string& path, const BaseReq& req,
132+
const std::map<std::string, std::string>& additional_headers,
133+
const std::map<std::string, std::string>& additional_params,
134+
const std::string& req_body, bool check_body, BaseResp* resp,
135+
const uint32_t &request_retry_num, bool is_ci_req = false);
136+
137+
CosResult DownloadRequest(const std::string& host, const std::string& path,
138+
const BaseReq& req, BaseResp* resp, std::ostream& os,
139+
const uint32_t &request_retry_num,
140+
const SharedTransferHandler& handler = nullptr);
141+
142+
CosResult UploadRequest(
143+
const std::string& host, const std::string& path, const BaseReq& req,
144+
const std::map<std::string, std::string>& additional_headers,
145+
const std::map<std::string, std::string>& additional_params,
146+
std::istream& is, BaseResp* resp, const uint32_t &request_retry_num, const SharedTransferHandler& handler = nullptr);
147+
148+
bool NoNeedRetry(const CosResult &result);
131149
};
132150

133151
} // namespace qcloud_cos

‎include/op/file_copy_task.h‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,19 @@
55
#include <map>
66
#include <string>
77

8-
#include "Poco/Foundation.h"
98
#include "Poco/Runnable.h"
109
#include "cos_defines.h"
10+
#include "util/base_op_util.h"
1111

1212
namespace qcloud_cos {
1313

1414
class FileCopyTask : public Poco::Runnable {
1515
public:
16-
FileCopyTask(const std::string& full_url, uint64_t conn_timeout_in_ms,
16+
FileCopyTask(const std::string& host,
17+
const std::string& path,
18+
const bool is_https,
19+
const BaseOpUtil& op_util,
20+
uint64_t conn_timeout_in_ms,
1721
uint64_t recv_timeout_in_ms);
1822

1923
~FileCopyTask() {}
@@ -45,7 +49,9 @@ class FileCopyTask : public Poco::Runnable {
4549
std::string GetLastModified() const { return m_last_modified; }
4650

4751
private:
48-
std::string m_full_url;
52+
std::string m_host;
53+
std::string m_path;
54+
bool m_is_https;
4955
std::map<std::string, std::string> m_headers;
5056
std::map<std::string, std::string> m_params;
5157
uint64_t m_conn_timeout_in_ms;
@@ -62,6 +68,10 @@ class FileCopyTask : public Poco::Runnable {
6268
std::string m_ca_location;
6369
SSLCtxCallback m_ssl_ctx_cb;
6470
void *m_user_data;
71+
72+
BaseOpUtil m_op_util;
73+
74+
void SendRequestOnce(std::string domain);
6575
};
6676

6777
} // namespace qcloud_cos

‎include/op/file_download_task.h‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,19 @@
1111
#include <map>
1212
#include <string>
1313

14-
#include "Poco/Foundation.h"
1514
#include "Poco/Runnable.h"
1615
#include "cos_config.h"
1716
#include "trsf/transfer_handler.h"
17+
#include "util/base_op_util.h"
1818

1919
namespace qcloud_cos {
2020

2121
class FileDownTask : public Poco::Runnable {
2222
public:
23-
FileDownTask(const std::string& full_url,
23+
FileDownTask(const std::string& host,
24+
const std::string& path,
25+
const bool is_https,
26+
const BaseOpUtil& op_util,
2427
const std::map<std::string, std::string>& headers,
2528
const std::map<std::string, std::string>& params,
2629
uint64_t conn_timeout_in_ms, uint64_t recv_timeout_in_ms,
@@ -57,7 +60,10 @@ class FileDownTask : public Poco::Runnable {
5760
std::string GetErrMsg() const { return m_err_msg; }
5861

5962
private:
60-
std::string m_full_url;
63+
std::string m_host;
64+
std::string m_path;
65+
bool m_is_https;
66+
BaseOpUtil m_op_util;
6167
std::map<std::string, std::string> m_headers;
6268
std::map<std::string, std::string> m_params;
6369
uint64_t m_conn_timeout_in_ms;
@@ -79,6 +85,8 @@ class FileDownTask : public Poco::Runnable {
7985
void *m_user_data;
8086

8187
SharedConfig m_config;
88+
89+
void SendRequestOnce(std::string domain);
8290
};
8391

8492
} // namespace qcloud_cos

‎include/op/file_upload_task.h‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,24 +3,31 @@
33
#include <map>
44
#include <string>
55

6-
#include "Poco/Foundation.h"
76
#include "Poco/Runnable.h"
87
#include "request/object_req.h"
98
#include "trsf/transfer_handler.h"
9+
#include "util/base_op_util.h"
1010

1111
namespace qcloud_cos {
1212

1313
class FileUploadTask : public Poco::Runnable {
1414
public:
15-
FileUploadTask(const std::string& full_url, uint64_t conn_timeout_in_ms,
15+
FileUploadTask(const std::string& host,
16+
const std::string& path,
17+
const bool is_https,
18+
const BaseOpUtil& op_util,
19+
uint64_t conn_timeout_in_ms,
1620
uint64_t recv_timeout_in_ms, unsigned char* pbuf = NULL,
1721
const size_t data_len = 0,
1822
bool verify_cert = true,
1923
const std::string& ca_location = "",
2024
SSLCtxCallback ssl_ctx_cb = nullptr,
2125
void *user_data = nullptr);
2226

23-
FileUploadTask(const std::string& full_url,
27+
FileUploadTask(const std::string& host,
28+
const std::string& path,
29+
const bool is_https,
30+
const BaseOpUtil& op_util,
2431
const std::map<std::string, std::string>& headers,
2532
const std::map<std::string, std::string>& params,
2633
uint64_t conn_timeout_in_ms, uint64_t recv_timeout_in_ms,
@@ -30,7 +37,10 @@ class FileUploadTask : public Poco::Runnable {
3037
SSLCtxCallback ssl_ctx_cb = nullptr,
3138
void *user_data = nullptr);
3239

33-
FileUploadTask(const std::string& full_url,
40+
FileUploadTask(const std::string& host,
41+
const std::string& path,
42+
const bool is_https,
43+
const BaseOpUtil& op_util,
3444
const std::map<std::string, std::string>& headers,
3545
const std::map<std::string, std::string>& params,
3646
uint64_t conn_timeout_in_ms, uint64_t recv_timeout_in_ms,
@@ -91,7 +101,9 @@ class FileUploadTask : public Poco::Runnable {
91101
}
92102

93103
private:
94-
std::string m_full_url;
104+
std::string m_host;
105+
std::string m_path;
106+
bool m_is_https;
95107
std::map<std::string, std::string> m_headers;
96108
std::map<std::string, std::string> m_params;
97109
uint64_t m_conn_timeout_in_ms;
@@ -115,6 +127,10 @@ class FileUploadTask : public Poco::Runnable {
115127

116128
bool mb_check_crc64;
117129
uint64_t m_crc64_value;
130+
131+
BaseOpUtil m_op_util;
132+
133+
void SendRequestOnce(std::string domain, std::string md5_str);
118134
};
119135

120136
} // namespace qcloud_cos

‎include/util/base_op_util.h‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#ifndef COS_CPP_SDK_V5_INCLUDE_UTIL_BASE_OP_UTIL_H_
2+
#define COS_CPP_SDK_V5_INCLUDE_UTIL_BASE_OP_UTIL_H_
3+
4+
#include <utility>
5+
6+
#include "cos_config.h"
7+
#include "op/cos_result.h"
8+
9+
namespace qcloud_cos {
10+
class BaseOpUtil {
11+
public:
12+
explicit BaseOpUtil(SharedConfig cos_conf) : m_config(std::move(cos_conf)) {}
13+
14+
BaseOpUtil() = default;
15+
16+
bool ShouldChangeBackupDomain(const CosResult &result, const uint32_t &request_num, bool is_ci_req = false) const;
17+
18+
void SleepBeforeRetry(const uint32_t &request_num) const;
19+
20+
std::string GetRealUrl(const std::string& host, const std::string& path, bool is_https, bool is_generate_presigned_url = false) const;
21+
22+
uint64_t GetMaxRetryTimes() const;
23+
24+
static std::string ChangeHostSuffix(const std::string& host);
25+
26+
private:
27+
SharedConfig m_config;
28+
29+
bool UseDefaultDomain() const;
30+
};
31+
} // namespace qcloud_cos
32+
#endif // COS_CPP_SDK_V5_INCLUDE_UTIL_BASE_OP_UTIL_H_

‎include/util/http_sender.h‎

Lines changed: 6 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -36,22 +36,6 @@ class HttpSender {
3636
SSLCtxCallback ssl_ctx_cb = nullptr,
3737
void *user_data = nullptr);
3838

39-
static int SendRequest(const SharedTransferHandler& handler,
40-
const std::string& http_method,
41-
const std::string& url_str,
42-
const std::map<std::string, std::string>& req_params,
43-
const std::map<std::string, std::string>& req_headers,
44-
const std::string& req_body,
45-
uint64_t conn_timeout_in_ms,
46-
uint64_t recv_timeout_in_ms,
47-
std::map<std::string, std::string>* resp_headers,
48-
std::ostream& resp_stream, std::string* err_msg,
49-
bool is_check_md5 = false,
50-
bool is_verify_cert = true,
51-
const std::string& ca_location = "",
52-
SSLCtxCallback ssl_ctx_cb = nullptr,
53-
void *user_data = nullptr);
54-
5539
static int SendRequest(const SharedTransferHandler& handler,
5640
const std::string& http_method,
5741
const std::string& url_str,
@@ -72,16 +56,19 @@ class HttpSender {
7256
const std::string& url_str,
7357
const std::map<std::string, std::string>& req_params,
7458
const std::map<std::string, std::string>& req_headers,
75-
std::istream& is, uint64_t conn_timeout_in_ms,
59+
std::istream& is,
60+
uint64_t conn_timeout_in_ms,
7661
uint64_t recv_timeout_in_ms,
7762
std::map<std::string, std::string>* resp_headers,
78-
std::ostream& resp_stream, std::string* err_msg,
63+
std::ostream& resp_stream,
64+
std::string* err_msg,
7965
bool is_check_md5 = false,
8066
bool is_verify_cert = true,
8167
const std::string& ca_location = "",
8268
SSLCtxCallback ssl_ctx_cb = nullptr,
8369
void *user_data = nullptr,
84-
const char *req_body_buf = nullptr, size_t req_body_len = 0);
70+
const char *req_body_buf = nullptr,
71+
size_t req_body_len = 0);
8572

8673
static int SendRequest(const SharedTransferHandler& handler,
8774
const std::string& http_method,

0 commit comments

Comments
 (0)