Skip to content

Commit 3883f04

Browse files
sjperkinssjperkins
committed
Update to more recent version of the AWS C, CRT and C++ SDK's
1 parent 6b69c72 commit 3883f04

File tree

33 files changed

+921
-186
lines changed

33 files changed

+921
-186
lines changed

‎tensorstore/kvstore/s3_sdk/localstack_test.cc‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -300,7 +300,11 @@ class LocalStackFixture : public ::testing::Test {
300300
auto cfg = Aws::Client::ClientConfiguration{};
301301
cfg.endpointOverride = endpoint_url();
302302
cfg.region = Region();
303-
auto create_client = std::make_shared<Aws::S3::S3Client>(Aws::Auth::AWSCredentials(), cfg);
303+
auto create_client = std::make_shared<Aws::S3::S3Client>(
304+
Aws::Auth::AWSCredentials(),
305+
cfg,
306+
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always,
307+
false);
304308

305309
auto create_request = Aws::S3::Model::CreateBucketRequest{};
306310
create_request.SetBucket(Bucket());

‎tensorstore/kvstore/s3_sdk/s3_context.cc‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616

1717
#include <cstdarg>
1818
#include <memory>
19+
#include <string_view>
20+
#include <vector>
1921

2022
#include <aws/core/Aws.h>
2123
#include <aws/core/auth/AWSCredentialsProviderChain.h>
@@ -102,7 +104,7 @@ class CustomHttpClient : public AwsHttpClient {
102104
std::vector<char> buffer(absl::CordBuffer::kDefaultLimit);
103105
std::streampos original = body->tellg();
104106
while (body->read(buffer.data(), buffer.size()) || body->gcount() > 0) {
105-
payload.Append(absl::string_view(buffer.data(), body->gcount()));
107+
payload.Append(std::string_view(buffer.data(), body->gcount()));
106108
}
107109

108110
if(payload.size() > k1MB) {
@@ -228,23 +230,35 @@ class AWSLogSystem : public AwsLogSystemInterface {
228230
void Log(AwsLogLevel log_level, const char* tag,
229231
const char* format, ...) override;
230232

233+
// Overridden, but prefer the safer LogStream
234+
void vaLog(AwsLogLevel log_level, const char* tag,
235+
const char* format, va_list args) override;
236+
231237
private:
232-
void LogMessage(AwsLogLevel log_level, const std::string & message);
238+
void LogMessage(AwsLogLevel log_level, std::string_view message);
233239
AwsLogLevel log_level_;
234240
};
235241

236-
237242
void AWSLogSystem::Log(AwsLogLevel log_level, const char* tag,
238243
const char* format, ...) {
239-
char buffer[256];
244+
// https://www.open-std.org/JTC1/SC22/WG14/www/docs/n1570.pdf
245+
// Section 7.16
240246
va_list args;
241247
va_start(args, format);
242-
vsnprintf(buffer, 256, format, args);
248+
vaLog(log_level, tag, format, args);
243249
va_end(args);
250+
}
251+
252+
void AWSLogSystem::vaLog(AwsLogLevel log_level, const char* tag,
253+
const char* format, va_list args) {
254+
// https://www.open-std.org/JTC1/SC22/WG14/www/docs/n1570.pdf
255+
// Section 7.16
256+
char buffer[256];
257+
vsnprintf(buffer, 256, format, args);
244258
LogMessage(log_level, buffer);
245259
}
246260

247-
void AWSLogSystem::LogMessage(AwsLogLevel log_level, const std::string & message) {
261+
void AWSLogSystem::LogMessage(AwsLogLevel log_level, std::string_view message) {
248262
switch(log_level) {
249263
case AwsLogLevel::Info:
250264
ABSL_LOG(INFO) << message;

‎third_party/com_github_aws_c_auth/aws_c_auth.BUILD.bazel‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Description:
2+
# AWS C Auth
3+
4+
package(default_visibility = ["//visibility:public"])
5+
6+
licenses(["notice"]) # Apache 2.0
7+
8+
exports_files(["LICENSE"])
9+
10+
cc_library(
11+
name = "aws_c_auth",
12+
srcs = glob([
13+
"include/aws/auth/**/*.h",
14+
"source/*.c",
15+
]),
16+
includes = ["include"],
17+
deps = [
18+
"@com_github_aws_c_http//:aws_c_http",
19+
"@com_github_aws_c_sdkutils//:aws_c_sdkutils",
20+
],
21+
)

‎third_party/com_github_aws_c_auth/workspace.bzl‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Copyright 2024 The TensorStore Authors
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
load("//third_party:repo.bzl", "third_party_http_archive")
16+
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
17+
18+
def repo():
19+
maybe(
20+
third_party_http_archive,
21+
name = "com_github_aws_c_auth",
22+
sha256 = "f249a12a6ac319e929c005fb7efd5534c83d3af3a3a53722626ff60a494054bb",
23+
strip_prefix = "aws-c-auth-0.7.22",
24+
urls = [
25+
"https://github.com/awslabs/aws-c-auth/archive/refs/tags/v0.7.22.tar.gz",
26+
],
27+
build_file = Label("//third_party:com_github_aws_c_auth/aws_c_auth.BUILD.bazel"),
28+
system_build_file = Label("//third_party:com_github_aws_c_auth/system.BUILD.bazel"),
29+
cmake_name = "aws_c_auth",
30+
cmake_target_mapping = {
31+
"@com_github_aws_c_auth//:aws_c_auth": "aws_c_auth::aws_c_auth",
32+
},
33+
bazel_to_cmake = {},
34+
)

‎third_party/com_github_aws_c_cal/aws_c_cal.BUILD.bazel‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Description:
2+
# AWS s2n tls
3+
4+
package(default_visibility = ["//visibility:public"])
5+
6+
licenses(["notice"]) # Apache 2.0
7+
8+
cc_library(
9+
name = "aws_c_cal",
10+
srcs = glob([
11+
"include/aws/cal/*.h",
12+
"include/aws/cal/private/*.h",
13+
"source/*.c"
14+
]) + select({
15+
"@platforms//os:windows": glob([
16+
"source/windows/*.c",
17+
]),
18+
"@platforms//os:linux": glob([
19+
"source/unix/*.c",
20+
]),
21+
"@platforms//os:osx": glob([
22+
"source/darwin/*.c",
23+
])
24+
}),
25+
includes = ["include"],
26+
deps = [
27+
"@com_github_aws_c_common//:aws_c_common",
28+
"@com_google_boringssl//:crypto"
29+
]
30+
)

‎third_party/com_github_aws_c_cal/workspace.bzl‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Copyright 2024 The TensorStore Authors
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
load("//third_party:repo.bzl", "third_party_http_archive")
16+
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
17+
18+
19+
def repo():
20+
maybe(
21+
third_party_http_archive,
22+
name = "com_github_aws_c_cal",
23+
sha256 = "9c51afbece6aa7a4a3e40b99c242884c1744d7f949a3f720cea41d247ac2d06a",
24+
strip_prefix = "aws-c-cal-0.7.0",
25+
urls = [
26+
"https://github.com/awslabs/aws-c-cal/archive/refs/tags/v0.7.0.tar.gz",
27+
],
28+
build_file = Label("//third_party:com_github_aws_c_cal/aws_c_cal.BUILD.bazel"),
29+
#system_build_file = Label("//third_party:com_github_aws_c_cal/system.BUILD.bazel"),
30+
cmake_name = "aws_c_cal",
31+
cmake_target_mapping = {
32+
"@com_github_aws_c_cal//:aws_c_cal": "aws_c_cal::aws_c_cal",
33+
},
34+
bazel_to_cmake = {},
35+
)

‎third_party/com_github_aws_c_common/aws_c_common.BUILD.bazel‎

Lines changed: 64 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,96 @@
11
# Description:
22
# AWS C Common
33

4+
load("@bazel_skylib//rules:write_file.bzl", "write_file")
5+
46
package(default_visibility = ["//visibility:public"])
57

68
licenses(["notice"]) # Apache 2.0
79

810
exports_files(["LICENSE"])
911

12+
write_file(
13+
name = "write_config_h",
14+
out = "include/aws/common/config.h",
15+
newline = "auto",
16+
17+
content = [
18+
"#ifndef AWS_COMMON_CONFIG_H",
19+
"#define AWS_COMMON_CONFIG_H",
20+
"",
21+
"#define AWS_HAVE_GCC_OVERFLOW_MATH_EXTENSIONS",
22+
"#define AWS_HAVE_GCC_INLINE_ASM",
23+
"#define AWS_HAVE_POSIX_LARGE_FILE_SUPPORT",
24+
] + select({
25+
"@platforms//os:linux": [
26+
"#define AWS_HAVE_EXECINFO",
27+
"#define AWS_HAVE_LINUX_IF_LINK_H",
28+
],
29+
"@platforms//os:windows": [
30+
"#define AWS_HAVE_WINAPI_DESKTOP",
31+
],
32+
}) + [
33+
"#undef AWS_HAVE_WINAPI_DESKTOP",
34+
# TODO: improve with logic from AwsSIMD.cmake
35+
# but this strictly requires configure style tests...
36+
"#undef AWS_USE_CPU_EXTENSIONS",
37+
"#undef AWS_HAVE_MSVC_INTRINSICS_X64",
38+
"#undef AWS_HAVE_AVX2_INTRINSICS",
39+
"#undef AWS_HAVE_AVX512_INTRINSICS",
40+
"#undef AWS_HAVE_MM256_EXTRACT_EPI64",
41+
"#undef AWS_HAVE_CLMUL",
42+
"#undef AWS_HAVE_ARM32_CRC",
43+
"#undef AWS_HAVE_ARMv8_1",
44+
"#undef AWS_ARCH_ARM64",
45+
"#undef AWS_ARCH_INTEL",
46+
"#undef AWS_ARCH_INTEL_X64",
47+
"",
48+
"#endif"
49+
]
50+
)
51+
1052
cc_library(
1153
name = "aws_c_common",
1254
srcs = glob([
1355
"include/aws/common/*.h",
56+
"include/aws/common/external/*.h",
1457
"include/aws/common/private/*.h",
1558
"source/*.c",
59+
"source/arch/generic/*.c",
60+
"source/external/**/*.h",
61+
"source/external/**/*.c",
1662
]) + select({
1763
"@platforms//os:windows": glob([
1864
"source/windows/*.c",
1965
]),
20-
"//conditions:default": glob([
66+
"@platforms//os:linux": glob([
67+
"source/linux/*.c",
2168
"source/posix/*.c",
2269
]),
70+
"@platforms//os:osx": glob([
71+
"source/posix/*.c",
72+
])
2373
}),
24-
hdrs = [
25-
"include/aws/common/config.h",
74+
hdrs = glob([
75+
"include/aws/common/*.h",
76+
"include/aws/common/private/*.h",
77+
]) + [
78+
":write_config_h"
79+
],
80+
defines = [
81+
# TODO: improve this with logic from AwsThreadAffinity.cmake
82+
"AWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_NONE",
83+
# Disable macro tracing API
84+
"INTEL_NO_ITTNOTIFY_API",
2685
],
27-
defines = [],
2886
includes = [
2987
"include",
88+
"source/external",
89+
"source/external/libcbor"
3090
],
3191
textual_hdrs = glob([
3292
"include/**/*.inl",
3393
]),
3494
deps = [],
3595
)
3696

37-
genrule(
38-
name = "config_h",
39-
srcs = [
40-
"include/aws/common/config.h.in",
41-
],
42-
outs = [
43-
"include/aws/common/config.h",
44-
],
45-
cmd = "sed 's/cmakedefine/undef/g' $< > $@",
46-
)

‎third_party/com_github_aws_c_common/workspace.bzl‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,10 @@ def repo():
1919
maybe(
2020
third_party_http_archive,
2121
name = "com_github_aws_c_common",
22-
sha256 = "01c2a58553a37b3aa5914d9e0bf7bf14507ff4937bc5872a678892ca20fcae1f",
23-
strip_prefix = "aws-c-common-0.4.29",
22+
sha256 = "adf838daf6a60aa31268522105b03262d745f529bc981d3ac665424133d6f91b",
23+
strip_prefix = "aws-c-common-0.9.23",
2424
urls = [
25-
"https://storage.googleapis.com/mirror.tensorflow.org/github.com/awslabs/aws-c-common/archive/v0.4.29.tar.gz",
26-
"https://github.com/awslabs/aws-c-common/archive/v0.4.29.tar.gz",
25+
"https://github.com/awslabs/aws-c-common/archive/v0.9.23.tar.gz",
2726
],
2827
build_file = Label("//third_party:com_github_aws_c_common/aws_c_common.BUILD.bazel"),
2928
system_build_file = Label("//third_party:com_github_aws_c_common/system.BUILD.bazel"),

‎third_party/com_github_aws_c_compression/aws_c_compression.BUILD.bazel‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# Description:
2+
# AWS C Compression
3+
4+
package(default_visibility = ["//visibility:public"])
5+
6+
licenses(["notice"]) # Apache 2.0
7+
8+
exports_files(["LICENSE"])
9+
10+
cc_library(
11+
name = "aws_c_compression",
12+
srcs = glob([
13+
"include/aws/compression/**/*.h",
14+
"source/*.c",
15+
]),
16+
includes = ["include"],
17+
deps = [
18+
"@com_github_aws_c_common//:aws_c_common",
19+
],
20+
)

‎third_party/com_github_aws_c_compression/workspace.bzl‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Copyright 2024 The TensorStore Authors
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
load("//third_party:repo.bzl", "third_party_http_archive")
16+
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
17+
18+
def repo():
19+
maybe(
20+
third_party_http_archive,
21+
name = "com_github_aws_c_compression",
22+
sha256 = "517c361f3b7fffca08efd5ad251a20489794f056eab0dfffacc6d5b341df8e86",
23+
strip_prefix = "aws-c-compression-0.2.18",
24+
urls = [
25+
"https://github.com/awslabs/aws-c-compression/archive/v0.2.18.tar.gz",
26+
],
27+
build_file = Label("//third_party:com_github_aws_c_compression/aws_c_compression.BUILD.bazel"),
28+
system_build_file = Label("//third_party:com_github_aws_c_compression/system.BUILD.bazel"),
29+
cmake_name = "aws_c_compression",
30+
cmake_target_mapping = {
31+
"@com_github_aws_c_compression//:aws_c_compression": "aws_c_compression::aws_c_compression",
32+
},
33+
bazel_to_cmake = {},
34+
)

0 commit comments

Comments
 (0)