Skip to content

Commit b19b56b

Browse files
author
zackcao
committed
Merge branch 'dev-auto-retry-request-advanced' into 'master' (merge request !60)
[合规重试]高级接口请求支持重试策略
2 parents 918a246 + 3f6afa8 commit b19b56b

File tree

7 files changed

+245
-148
lines changed

7 files changed

+245
-148
lines changed

‎include/op/file_copy_task.h‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,19 @@
55
#include <map>
66
#include <string>
77

8-
#include "Poco/Foundation.h"
98
#include "Poco/Runnable.h"
109
#include "cos_defines.h"
10+
#include "util/base_op_util.h"
1111

1212
namespace qcloud_cos {
1313

1414
class FileCopyTask : public Poco::Runnable {
1515
public:
16-
FileCopyTask(const std::string& full_url, uint64_t conn_timeout_in_ms,
16+
FileCopyTask(const std::string& host,
17+
const std::string& path,
18+
const bool is_https,
19+
const BaseOpUtil& op_util,
20+
uint64_t conn_timeout_in_ms,
1721
uint64_t recv_timeout_in_ms);
1822

1923
~FileCopyTask() {}
@@ -45,7 +49,9 @@ class FileCopyTask : public Poco::Runnable {
4549
std::string GetLastModified() const { return m_last_modified; }
4650

4751
private:
48-
std::string m_full_url;
52+
std::string m_host;
53+
std::string m_path;
54+
bool m_is_https;
4955
std::map<std::string, std::string> m_headers;
5056
std::map<std::string, std::string> m_params;
5157
uint64_t m_conn_timeout_in_ms;
@@ -62,6 +68,10 @@ class FileCopyTask : public Poco::Runnable {
6268
std::string m_ca_location;
6369
SSLCtxCallback m_ssl_ctx_cb;
6470
void *m_user_data;
71+
72+
BaseOpUtil m_op_util;
73+
74+
void SendRequestOnce(std::string domain);
6575
};
6676

6777
} // namespace qcloud_cos

‎include/op/file_download_task.h‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,19 @@
1111
#include <map>
1212
#include <string>
1313

14-
#include "Poco/Foundation.h"
1514
#include "Poco/Runnable.h"
1615
#include "cos_config.h"
1716
#include "trsf/transfer_handler.h"
17+
#include "util/base_op_util.h"
1818

1919
namespace qcloud_cos {
2020

2121
class FileDownTask : public Poco::Runnable {
2222
public:
23-
FileDownTask(const std::string& full_url,
23+
FileDownTask(const std::string& host,
24+
const std::string& path,
25+
const bool is_https,
26+
const BaseOpUtil& op_util,
2427
const std::map<std::string, std::string>& headers,
2528
const std::map<std::string, std::string>& params,
2629
uint64_t conn_timeout_in_ms, uint64_t recv_timeout_in_ms,
@@ -57,7 +60,10 @@ class FileDownTask : public Poco::Runnable {
5760
std::string GetErrMsg() const { return m_err_msg; }
5861

5962
private:
60-
std::string m_full_url;
63+
std::string m_host;
64+
std::string m_path;
65+
bool m_is_https;
66+
BaseOpUtil m_op_util;
6167
std::map<std::string, std::string> m_headers;
6268
std::map<std::string, std::string> m_params;
6369
uint64_t m_conn_timeout_in_ms;
@@ -79,6 +85,8 @@ class FileDownTask : public Poco::Runnable {
7985
void *m_user_data;
8086

8187
SharedConfig m_config;
88+
89+
void SendRequestOnce(std::string domain);
8290
};
8391

8492
} // namespace qcloud_cos

‎include/op/file_upload_task.h‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,24 +3,31 @@
33
#include <map>
44
#include <string>
55

6-
#include "Poco/Foundation.h"
76
#include "Poco/Runnable.h"
87
#include "request/object_req.h"
98
#include "trsf/transfer_handler.h"
9+
#include "util/base_op_util.h"
1010

1111
namespace qcloud_cos {
1212

1313
class FileUploadTask : public Poco::Runnable {
1414
public:
15-
FileUploadTask(const std::string& full_url, uint64_t conn_timeout_in_ms,
15+
FileUploadTask(const std::string& host,
16+
const std::string& path,
17+
const bool is_https,
18+
const BaseOpUtil& op_util,
19+
uint64_t conn_timeout_in_ms,
1620
uint64_t recv_timeout_in_ms, unsigned char* pbuf = NULL,
1721
const size_t data_len = 0,
1822
bool verify_cert = true,
1923
const std::string& ca_location = "",
2024
SSLCtxCallback ssl_ctx_cb = nullptr,
2125
void *user_data = nullptr);
2226

23-
FileUploadTask(const std::string& full_url,
27+
FileUploadTask(const std::string& host,
28+
const std::string& path,
29+
const bool is_https,
30+
const BaseOpUtil& op_util,
2431
const std::map<std::string, std::string>& headers,
2532
const std::map<std::string, std::string>& params,
2633
uint64_t conn_timeout_in_ms, uint64_t recv_timeout_in_ms,
@@ -30,7 +37,10 @@ class FileUploadTask : public Poco::Runnable {
3037
SSLCtxCallback ssl_ctx_cb = nullptr,
3138
void *user_data = nullptr);
3239

33-
FileUploadTask(const std::string& full_url,
40+
FileUploadTask(const std::string& host,
41+
const std::string& path,
42+
const bool is_https,
43+
const BaseOpUtil& op_util,
3444
const std::map<std::string, std::string>& headers,
3545
const std::map<std::string, std::string>& params,
3646
uint64_t conn_timeout_in_ms, uint64_t recv_timeout_in_ms,
@@ -91,7 +101,9 @@ class FileUploadTask : public Poco::Runnable {
91101
}
92102

93103
private:
94-
std::string m_full_url;
104+
std::string m_host;
105+
std::string m_path;
106+
bool m_is_https;
95107
std::map<std::string, std::string> m_headers;
96108
std::map<std::string, std::string> m_params;
97109
uint64_t m_conn_timeout_in_ms;
@@ -115,6 +127,10 @@ class FileUploadTask : public Poco::Runnable {
115127

116128
bool mb_check_crc64;
117129
uint64_t m_crc64_value;
130+
131+
BaseOpUtil m_op_util;
132+
133+
void SendRequestOnce(std::string domain, std::string md5_str);
118134
};
119135

120136
} // namespace qcloud_cos

‎src/op/file_copy_task.cpp‎

Lines changed: 42 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,20 @@
44
#include "request/object_req.h"
55
#include "response/object_resp.h"
66
#include "util/http_sender.h"
7+
#include "util/base_op_util.h"
78

89
namespace qcloud_cos {
910

10-
FileCopyTask::FileCopyTask(const std::string& full_url,
11+
FileCopyTask::FileCopyTask(const std::string& host,
12+
const std::string& path,
13+
const bool is_https,
14+
const BaseOpUtil& op_util,
1115
uint64_t conn_timeout_in_ms,
1216
uint64_t recv_timeout_in_ms)
13-
: m_full_url(full_url),
17+
: m_host(host),
18+
m_path(path),
19+
m_is_https(is_https),
20+
m_op_util(op_util),
1421
m_conn_timeout_in_ms(conn_timeout_in_ms),
1522
m_recv_timeout_in_ms(recv_timeout_in_ms),
1623
m_is_task_success(false),
@@ -59,35 +66,53 @@ void FileCopyTask::run() {
5966
}
6067

6168
void FileCopyTask::CopyTask() {
62-
int loop = 0;
63-
do {
64-
loop++;
69+
std::string domain = m_host;
70+
for (int i = 0;; i++) {
71+
SendRequestOnce(domain);
72+
if (i >= m_op_util.GetMaxRetryTimes()) {
73+
break;
74+
}
75+
if (m_is_task_success) {
76+
break;
77+
}
78+
SDK_LOG_ERR("FileCopy: host(%s) path(%s) fail, retry num: %d, httpcode:%d, resp: %s",
79+
domain.c_str(), m_path.c_str(), i, m_http_status, m_resp.c_str());
80+
if (m_http_status >= 400 && m_http_status < 500) {
81+
break;
82+
}
83+
CosResult result;
84+
result.SetHttpStatus(m_http_status);
85+
result.ParseFromHttpResponse(m_resp_headers, m_resp);
86+
if (m_op_util.ShouldChangeBackupDomain(result, i)) {
87+
domain = m_op_util.ChangeHostSuffix(domain);
88+
}
89+
m_op_util.SleepBeforeRetry(i);
90+
}
91+
}
92+
93+
void FileCopyTask::SendRequestOnce(std::string domain) {
6594
m_resp_headers.clear();
6695
m_resp = "";
6796

68-
m_http_status = HttpSender::SendRequest(nullptr,
69-
"PUT", m_full_url, m_params, m_headers, "", m_conn_timeout_in_ms,
70-
m_recv_timeout_in_ms, &m_resp_headers, &m_resp, &m_err_msg,
71-
false, m_verify_cert, m_ca_location, m_ssl_ctx_cb, m_user_data);
97+
std::string full_url = m_op_util.GetRealUrl(domain, m_path, m_is_https);
98+
m_http_status = HttpSender::SendRequest(nullptr, "PUT", full_url, m_params, m_headers, "", m_conn_timeout_in_ms,
99+
m_recv_timeout_in_ms, &m_resp_headers, &m_resp, &m_err_msg, false, m_verify_cert, m_ca_location, m_ssl_ctx_cb,
100+
m_user_data);
72101

73102
if (m_http_status != 200) {
74-
SDK_LOG_ERR("FileUpload: url(%s) fail, httpcode:%d, resp: %s",
75-
m_full_url.c_str(), m_http_status, m_resp.c_str());
76-
m_is_task_success = false;
77-
continue;
103+
m_is_task_success = false;
104+
return;
78105
}
79106

80107
UploadPartCopyDataResp resp;
81108
if (!resp.ParseFromXmlString(m_resp)) {
82-
SDK_LOG_ERR("FileUpload response string is illegal. try again.");
83-
m_is_task_success = false;
84-
continue;
109+
m_is_task_success = false;
110+
return;
85111
}
86112

87113
m_etag = resp.GetEtag();
88114
m_last_modified = resp.GetLastModified();
89115
m_is_task_success = true;
90-
} while (!m_is_task_success && loop <= kMaxRetryTimes);
91116
}
92117

93118
} // namespace qcloud_cos

‎src/op/file_download_task.cpp‎

Lines changed: 50 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,14 @@
55
#include <sstream>
66
#include "cos_sys_config.h"
77
#include "util/http_sender.h"
8+
#include "util/base_op_util.h"
89

910
namespace qcloud_cos {
1011

11-
FileDownTask::FileDownTask(const std::string& full_url,
12+
FileDownTask::FileDownTask(const std::string& host,
13+
const std::string& path,
14+
const bool is_https,
15+
const BaseOpUtil& op_util,
1216
const std::map<std::string, std::string>& headers,
1317
const std::map<std::string, std::string>& params,
1418
uint64_t conn_timeout_in_ms,
@@ -20,7 +24,10 @@ FileDownTask::FileDownTask(const std::string& full_url,
2024
const std::string& ca_lication,
2125
SSLCtxCallback ssl_ctx_cb,
2226
void *user_data)
23-
: m_full_url(full_url),
27+
: m_host(host),
28+
m_path(path),
29+
m_is_https(is_https),
30+
m_op_util(op_util),
2431
m_headers(headers),
2532
m_params(params),
2633
m_conn_timeout_in_ms(conn_timeout_in_ms),
@@ -76,39 +83,49 @@ std::map<std::string, std::string> FileDownTask::GetRespHeaders() {
7683
}
7784

7885
void FileDownTask::DownTask() {
79-
char range_head[128];
80-
memset(range_head, 0, sizeof(range_head));
81-
snprintf(range_head, sizeof(range_head), "bytes=%" PRIu64 "-%" PRIu64, m_offset,
82-
(m_offset + m_data_len - 1));
83-
84-
// 增加Range头域，避免大文件时将整个文件下载
85-
m_headers["Range"] = range_head;
86-
87-
int try_times = 0;
88-
do {
89-
try_times++;
90-
//if (m_handler) {
91-
// SDK_LOG_INFO("transfer send GET request");
92-
// std::istringstream iss("");
93-
// std::ostringstream oss;
94-
// m_http_status = HttpSender::TransferSendRequest(
95-
// m_handler, "GET", m_full_url, m_params, m_headers, iss,
96-
// m_conn_timeout_in_ms, m_recv_timeout_in_ms, &m_resp_headers, oss,
97-
// &m_err_msg, false);
98-
// m_resp = oss.str();
99-
//} else {
100-
m_http_status = HttpSender::SendRequest(
101-
m_handler, "GET", m_full_url, m_params, m_headers, "",
102-
m_conn_timeout_in_ms, m_recv_timeout_in_ms, &m_resp_headers, &m_resp,
103-
&m_err_msg, false, m_verify_cert, m_ca_location, m_ssl_ctx_cb, m_user_data);
86+
char range_head[128];
87+
memset(range_head, 0, sizeof(range_head));
88+
snprintf(range_head, sizeof(range_head), "bytes=%" PRIu64 "-%" PRIu64, m_offset, (m_offset + m_data_len - 1));
89+
90+
// 增加Range头域，避免大文件时将整个文件下载
91+
m_headers["Range"] = range_head;
92+
93+
std::string domain = m_host;
94+
for (int i = 0;; i++) {
95+
SendRequestOnce(domain);
96+
if (i >= m_op_util.GetMaxRetryTimes()) {
97+
break;
98+
}
99+
if (m_is_task_success) {
100+
break;
101+
}
102+
SDK_LOG_ERR("FileDownload: host(%s) path(%s) fail, httpcode:%d, resp: %s, try_times: %d", domain.c_str(),
103+
m_path.c_str(), m_http_status, m_resp.c_str(), i);
104+
if (m_http_status >= 400 && m_http_status < 500) {
105+
break;
106+
}
107+
CosResult result;
108+
result.SetHttpStatus(m_http_status);
109+
result.ParseFromHttpResponse(m_resp_headers, m_resp);
110+
if (m_op_util.ShouldChangeBackupDomain(result, i)) {
111+
domain = m_op_util.ChangeHostSuffix(domain);
112+
}
113+
m_op_util.SleepBeforeRetry(i);
114+
}
115+
return;
116+
}
117+
118+
void FileDownTask::SendRequestOnce(std::string domain) {
119+
std::string full_url = m_op_util.GetRealUrl(domain, m_path, m_is_https);
120+
m_http_status = HttpSender::SendRequest(m_handler, "GET", full_url, m_params, m_headers, "", m_conn_timeout_in_ms,
121+
m_recv_timeout_in_ms, &m_resp_headers, &m_resp, &m_err_msg, false, m_verify_cert, m_ca_location, m_ssl_ctx_cb,
122+
m_user_data);
104123
//}
105-
//当实际长度小于请求的数据长度时httpcode为206
124+
// 当实际长度小于请求的数据长度时httpcode为206
106125
if (m_http_status != 200 && m_http_status != 206) {
107-
SDK_LOG_ERR("FileDownload: url(%s) fail, httpcode:%d, resp: %s, try_times:%d",
108-
m_full_url.c_str(), m_http_status, m_resp.c_str(), try_times);
109-
m_is_task_success = false;
110-
m_real_down_len = 0;
111-
continue;
126+
m_is_task_success = false;
127+
m_real_down_len = 0;
128+
return;
112129
}
113130

114131
size_t buf_max_size = m_data_len;
@@ -117,10 +134,6 @@ void FileDownTask::DownTask() {
117134
m_real_down_len = len;
118135
m_is_task_success = true;
119136
m_resp = "";
120-
return;
121-
} while (!m_is_task_success && try_times <= kMaxRetryTimes);
122-
123-
return;
124137
}
125138

126139
} // namespace qcloud_cos

0 commit comments

Comments
 (0)