Skip to content

Commit 04bb0c2

Browse files
committed
feat: inline commit logic into settings packages
There are 30-100ms delays between each systemd service that runs during the settings config phase. Avoid some of these delays by running the settings commit logic as part of the subsequent config phase.
1 parent 0d79ff2 commit 04bb0c2

File tree

13 files changed

+173
-118
lines changed

13 files changed

+173
-118
lines changed

packages/os/pluto.service

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ Description=Generate additional settings for Kubernetes
44
# settings generators, and before settings-applier commits all the settings, renders
55
# config files, and restarts services.
66
After=network-online.target apiserver.service sundog.service
7-
Before=settings-applier.service
87
Requires=sundog.service
98
# We don't want to restart the unit if the network goes offline or apiserver restarts
109
Wants=network-online.target apiserver.service
@@ -15,16 +14,9 @@ RefuseManualStop=true
1514

1615
[Service]
1716
Type=oneshot
18-
# pluto needs access to any Kubernetes settings supplied through user-data, along with
19-
# network-related settings such as proxy servers. Commit any settings that might have
20-
# been generated during the sundog phase.
21-
ExecStartPre=/usr/bin/settings-committer
2217
ExecStart=/usr/bin/pluto
2318
RemainAfterExit=true
2419
StandardError=journal+console
2520

2621
[Install]
27-
# settings-applier requires sundog to succeed as a signal that all settings generators ran
28-
# successfully. Since pluto is an honorary settings generator, settings-applier also needs
29-
# it to succeed before it can start.
30-
RequiredBy=settings-applier.service
22+
RequiredBy=preconfigured.target

packages/os/settings-applier.service

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[Unit]
22
Description=Applies settings to create config files
3-
After=storewolf.service sundog.service early-boot-config.service apiserver.service
4-
Requires=storewolf.service sundog.service early-boot-config.service
3+
After=storewolf.service sundog.service pluto.service early-boot-config.service apiserver.service
4+
Requires=storewolf.service sundog.service pluto.service early-boot-config.service
55
# We don't want to restart the unit if apiserver restarts
66
Wants=apiserver.service
77
# Block manual interactions with this service, since it could leave the system in an
@@ -11,7 +11,6 @@ RefuseManualStop=true
1111

1212
[Service]
1313
Type=oneshot
14-
ExecStartPre=/usr/bin/settings-committer
1514
ExecStart=/usr/bin/thar-be-settings --all
1615
RemainAfterExit=true
1716
StandardError=journal+console

packages/os/sundog.service

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,6 @@ RefuseManualStop=true
1212

1313
[Service]
1414
Type=oneshot
15-
# Commit first so we can use settings from user data.
16-
ExecStartPre=/usr/bin/settings-committer
1715
ExecStart=/usr/bin/sundog
1816
RemainAfterExit=true
1917
StandardError=journal+console

sources/Cargo.lock

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

sources/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ models = { version = "0.1", path = "models" }
9494
parse-datetime = { version = "0.1", path = "parse-datetime" }
9595
pciclient = { version = "0.1", path = "pciclient" }
9696
retry-read = { version = "0.1", path = "retry-read" }
97+
settings-committer = { version = "0.1", path = "api/settings-committer" }
9798
signpost = { version = "0.1", path = "updater/signpost" }
9899
storewolf = { version = "0.1", path = "api/storewolf" }
99100
simple-settings-plugin = { version = "0.1", path = "api/simple-settings-plugin" }

sources/api/pluto/Cargo.toml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,12 @@ bottlerocket-modeled-types.workspace = true
2525
bottlerocket-settings-models.workspace = true
2626
constants.workspace = true
2727
imdsclient.workspace = true
28+
log.workspace = true
2829
rustls.workspace = true
2930
serde = { workspace = true, features = ["derive"] }
3031
serde_json.workspace = true
32+
settings-committer.workspace = true
33+
simplelog.workspace = true
3134
snafu.workspace = true
3235
tempfile.workspace = true
3336
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }

sources/api/pluto/src/main.rs

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,16 @@ Pluto returns a special exit code of 2 to inform `sundog` that a setting should
3030
example, if `max-pods` cannot be generated, we want `sundog` to skip it without failing since a
3131
reasonable default is available.
3232
*/
33+
#[macro_use]
34+
extern crate log;
3335

3436
mod api;
3537
mod aws;
3638
mod ec2;
3739
mod eks;
3840

3941
use api::{settings_view_get, settings_view_set, SettingsViewDelta};
42+
use simplelog::{Config as LogConfig, LevelFilter, SimpleLogger};
4043
use aws_smithy_experimental::hyper_1_0::CryptoMode;
4144
use base64::Engine;
4245
use bottlerocket_modeled_types::{KubernetesClusterDnsIp, KubernetesHostnameOverrideSource};
@@ -72,6 +75,7 @@ const PROVIDER: CryptoMode = CryptoMode::AwsLcFips;
7275

7376
mod error {
7477
use crate::{api, ec2, eks};
78+
use settings_committer::SettingsCommitterError;
7579
use snafu::Snafu;
7680
use std::net::AddrParseError;
7781

@@ -162,6 +166,12 @@ mod error {
162166

163167
#[snafu(display("Unable to create tempdir: {}", source))]
164168
Tempdir { source: std::io::Error },
169+
170+
#[snafu(display("Unable to commit pending transaction: {}", source))]
171+
Commit { source: SettingsCommitterError },
172+
173+
#[snafu(display("Logger setup error: {}", source))]
174+
Logger { source: log::SetLoggerError },
165175
}
166176
}
167177

@@ -488,6 +498,16 @@ fn set_aws_config(aws_k8s_info: &SettingsViewDelta, filepath: &Path) -> Result<(
488498
}
489499

490500
async fn run() -> Result<()> {
501+
// SimpleLogger will send errors to stderr and anything less to stdout.
502+
SimpleLogger::init(LevelFilter::Info, LogConfig::default()).context(error::LoggerSnafu)?;
503+
info!("Pluto started");
504+
505+
// pluto needs access to any Kubernetes settings supplied through user-data, along with
506+
// network-related settings such as proxy servers. Commit any settings that might have
507+
// been generated during the sundog phase.
508+
settings_committer::commit(constants::API_SOCKET, constants::LAUNCH_TRANSACTION).await.context(error::CommitSnafu)?;
509+
510+
info!("Retrieving settings values");
491511
let mut client = ImdsClient::new();
492512
let current_settings = api::get_aws_k8s_info().await.context(error::AwsInfoSnafu)?;
493513
let mut aws_k8s_info = SettingsViewDelta::from_api_response(current_settings);
Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#[macro_use]
2+
extern crate log;
3+
4+
use snafu::ResultExt;
5+
use std::{collections::HashMap};
6+
7+
const API_PENDING_URI_BASE: &str = "/v2/tx";
8+
const API_COMMIT_URI_BASE: &str = "/tx/commit";
9+
10+
pub mod error {
11+
use http::StatusCode;
12+
use snafu::Snafu;
13+
14+
/// Potential errors during user data management.
15+
#[derive(Debug, Snafu)]
16+
#[snafu(visibility(pub(super)))]
17+
pub enum SettingsCommitterError {
18+
#[snafu(display("Error sending {} to {}: {}", method, uri, source))]
19+
APIRequest {
20+
method: String,
21+
uri: String,
22+
#[snafu(source(from(apiclient::Error, Box::new)))]
23+
source: Box<apiclient::Error>,
24+
},
25+
26+
#[snafu(display("Error {} when sending {} to {}: {}", code, method, uri, response_body))]
27+
APIResponse {
28+
method: String,
29+
uri: String,
30+
code: StatusCode,
31+
response_body: String,
32+
},
33+
}
34+
}
35+
pub use error::SettingsCommitterError;
36+
pub type Result<T> = std::result::Result<T, error::SettingsCommitterError>;
37+
38+
/// Checks pending settings and logs them. We don't want to prevent a
39+
/// commit if there's a blip in retrieval or parsing of the pending
40+
/// settings. We know the system won't be functional without a commit,
41+
/// but we can live without logging what was committed.
42+
async fn check_pending_settings<S: AsRef<str>>(socket_path: S, transaction: &str) {
43+
let uri = format!("{API_PENDING_URI_BASE}?tx={transaction}");
44+
45+
debug!("GET-ing {uri} to determine if there are pending settings");
46+
let get_result = apiclient::raw_request(socket_path.as_ref(), &uri, "GET", None).await;
47+
let response_body = match get_result {
48+
Ok((code, response_body)) => {
49+
if !code.is_success() {
50+
warn!("Got {code} when sending GET to {uri}: {response_body}");
51+
return;
52+
}
53+
response_body
54+
}
55+
Err(err) => {
56+
warn!("Failed to GET pending settings from {uri}: {err}");
57+
return;
58+
}
59+
};
60+
61+
let pending_result: serde_json::Result<HashMap<String, serde_json::Value>> =
62+
serde_json::from_str(&response_body);
63+
match pending_result {
64+
Ok(pending) => {
65+
debug!("Pending settings for tx {}: {:?}", transaction, &pending);
66+
}
67+
Err(err) => {
68+
warn!("Failed to parse response from {uri}: {err}");
69+
}
70+
}
71+
}
72+
73+
/// Commits pending settings to live.
74+
async fn commit_pending_settings<S: AsRef<str>>(socket_path: S, transaction: &str) -> Result<()> {
75+
let uri = format!("{API_COMMIT_URI_BASE}?tx={transaction}");
76+
debug!("POST-ing to {uri} to move pending settings to live");
77+
78+
if let Err(e) = apiclient::raw_request(socket_path.as_ref(), &uri, "POST", None).await {
79+
match e {
80+
// Some types of response errors are OK for this use.
81+
apiclient::Error::ResponseStatus { code, body, .. } => {
82+
if code.as_u16() == 422 {
83+
info!("settings-committer found no settings changes to commit");
84+
return Ok(());
85+
} else {
86+
return error::APIResponseSnafu {
87+
method: "POST",
88+
uri,
89+
code,
90+
response_body: body,
91+
}
92+
.fail();
93+
}
94+
}
95+
// Any other type of error means we couldn't even make the request.
96+
_ => {
97+
return Err(e).context(error::APIRequestSnafu {
98+
method: "POST",
99+
uri,
100+
});
101+
}
102+
}
103+
}
104+
Ok(())
105+
}
106+
107+
pub async fn commit(socket_path: &str, transaction: &str) -> Result<()> {
108+
if log_enabled!(log::Level::Debug) {
109+
// We log the pending settings at Debug, so only fetch them if they won't be filtered.
110+
info!("Checking pending settings.");
111+
check_pending_settings(socket_path, transaction).await;
112+
}
113+
114+
info!("Committing settings.");
115+
commit_pending_settings(socket_path, transaction).await?;
116+
117+
Ok(())
118+
}

0 commit comments

Comments
 (0)