From d45f8fb99da7ed6d2c5bc0e17fa5e486c30b3279 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 14 Aug 2025 18:15:08 +0200 Subject: [PATCH 1/6] wip: add run-securityadmin job --- rust/operator-binary/src/controller.rs | 9 +- rust/operator-binary/src/controller/apply.rs | 3 + rust/operator-binary/src/controller/build.rs | 8 + .../src/controller/build/job_builder.rs | 185 ++++++++++++++++++ 4 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 rust/operator-binary/src/controller/build/job_builder.rs diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 4e23072..35fb559 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -5,10 +5,13 @@ use build::build; use snafu::{ResultExt, Snafu}; use stackable_operator::{ cluster_resources::ClusterResourceApplyStrategy, - commons::{affinity::StackableAffinity, product_image_selection::ProductImage}, + commons::{ + affinity::StackableAffinity, networking::DomainName, product_image_selection::ProductImage, + }, crd::listener::v1alpha1::Listener, k8s_openapi::api::{ apps::v1::StatefulSet, + batch::v1::Job, core::v1::{ConfigMap, Service, ServiceAccount}, policy::v1::PodDisruptionBudget, rbac::v1::RoleBinding, @@ -43,6 +46,7 @@ pub struct ContextNames { pub product_name: ProductName, pub operator_name: OperatorName, pub controller_name: ControllerName, + pub cluster_domain_name: DomainName, } pub struct Context { @@ -52,6 +56,7 @@ pub struct Context { impl Context { pub fn new(client: stackable_operator::client::Client, operator_name: OperatorName) -> Self { + let cluster_domain_name = client.kubernetes_cluster_info.cluster_domain.clone(); Context { client, names: ContextNames { @@ -60,6 +65,7 @@ impl Context { operator_name, controller_name: ControllerName::from_str("opensearchcluster") .expect("should be a valid controller name"), + cluster_domain_name, }, } } @@ -282,5 +288,6 @@ struct KubernetesResources { service_accounts: Vec, role_bindings: Vec, pod_disruption_budgets: Vec, + jobs: Vec, status: PhantomData, } diff --git a/rust/operator-binary/src/controller/apply.rs b/rust/operator-binary/src/controller/apply.rs index f0bd893..69a442f 100644 --- a/rust/operator-binary/src/controller/apply.rs +++ b/rust/operator-binary/src/controller/apply.rs @@ -72,6 +72,8 @@ impl<'a> Applier<'a> { let pod_disruption_budgets = self.add_resources(resources.pod_disruption_budgets).await?; + let jobs = self.add_resources(resources.jobs).await?; + self.cluster_resources .delete_orphaned_resources(self.client) .await @@ -85,6 +87,7 @@ impl<'a> Applier<'a> { service_accounts, role_bindings, pod_disruption_budgets, + jobs, status: PhantomData, }) } diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs index db8e35e..051f33d 100644 --- a/rust/operator-binary/src/controller/build.rs +++ b/rust/operator-binary/src/controller/build.rs @@ -2,8 +2,11 @@ use std::marker::PhantomData; use role_builder::RoleBuilder; +use crate::controller::build::job_builder::JobBuilder; + use super::{ContextNames, KubernetesResources, Prepared, ValidatedCluster}; +pub mod job_builder; pub mod node_config; pub mod role_builder; pub mod role_group_builder; @@ -13,8 +16,10 @@ pub fn build(names: &ContextNames, cluster: ValidatedCluster) -> KubernetesResou let mut stateful_sets = vec![]; let mut services = vec![]; let mut listeners = vec![]; + let mut jobs = vec![]; let role_builder = RoleBuilder::new(cluster.clone(), names); + let job_builder = JobBuilder::new(cluster.clone(), names); for role_group_builder in role_builder.role_group_builders() { config_maps.push(role_group_builder.build_config_map()); @@ -32,6 +37,8 @@ pub fn build(names: &ContextNames, cluster: ValidatedCluster) -> KubernetesResou let pod_disruption_budgets = role_builder.build_pdb().into_iter().collect(); + jobs.push(job_builder.build_run_securityadmin_job()); + KubernetesResources { stateful_sets, services, @@ -40,6 +47,7 @@ pub fn build(names: &ContextNames, cluster: ValidatedCluster) -> KubernetesResou service_accounts, role_bindings, pod_disruption_budgets, + jobs, status: PhantomData, } } diff --git a/rust/operator-binary/src/controller/build/job_builder.rs b/rust/operator-binary/src/controller/build/job_builder.rs new file mode 100644 index 0000000..ee2beb7 --- /dev/null +++ b/rust/operator-binary/src/controller/build/job_builder.rs @@ -0,0 +1,185 @@ +use stackable_operator::{ + builder::{ + meta::ObjectMetaBuilder, + pod::{container::ContainerBuilder, resources::ResourceRequirementsBuilder}, + }, + k8s_openapi::api::{ + batch::v1::{Job, JobSpec}, + core::v1::{ + PodSecurityContext, PodSpec, PodTemplateSpec, SecretVolumeSource, Volume, VolumeMount, + }, + }, + kube::api::ObjectMeta, + kvp::{ + Label, Labels, + consts::{STACKABLE_VENDOR_KEY, STACKABLE_VENDOR_VALUE}, + }, +}; + +use crate::{ + controller::{ContextNames, ValidatedCluster}, + framework::{ + IsLabelValue, builder::meta::ownerreference_from_resource, role_utils::ResourceNames, + }, +}; + +const RUN_SECURITYADMIN_CERT_VOLUME_NAME: &str = "tls"; +const RUN_SECURITYADMIN_CERT_VOLUME_MOUNT: &str = "/stackable/cert"; +const SECURITY_CONFIG_VOLUME_NAME: &str = "security-config"; +const SECURITY_CONFIG_VOLUME_MOUNT: &str = "/stackable/opensearch/config/opensearch-security"; +const RUN_SECURITYADMIN_CONTAINER_NAME: &str = "run-securityadmin"; + +pub struct JobBuilder<'a> { + cluster: ValidatedCluster, + context_names: &'a ContextNames, + resource_names: ResourceNames, +} + +impl<'a> JobBuilder<'a> { + pub fn new(cluster: ValidatedCluster, context_names: &'a ContextNames) -> JobBuilder<'a> { + JobBuilder { + cluster: cluster.clone(), + context_names, + resource_names: ResourceNames { + cluster_name: cluster.name.clone(), + product_name: context_names.product_name.clone(), + }, + } + } + + pub fn build_run_securityadmin_job(&self) -> Job { + let product_image = self + .cluster + .image + .resolve("opensearch", crate::built_info::PKG_VERSION); + // Maybe add a suffix for consecutive + let metadata = self.common_metadata(format!( + "{}-run-securityadmin", + self.resource_names.cluster_name, + )); + + let args = [ + "plugins/opensearch-security/tools/securityadmin.sh".to_string(), + "-cacert".to_string(), + "config/tls-client/ca.crt".to_string(), + "-cert".to_string(), + "config/tls-client/tls.crt".to_string(), + "-key".to_string(), + "config/tls-client/tls.key".to_string(), + "--hostname".to_string(), + self.opensearch_master_fqdn(), + "--configdir".to_string(), + "config/opensearch-security/".to_string(), + ]; + let mut cb = ContainerBuilder::new(RUN_SECURITYADMIN_CONTAINER_NAME) + .expect("should be a valid container name"); + let container = cb + .image_from_product_image(&product_image) + .command(vec!["sh".to_string(), "-c".to_string()]) + .args(vec![args.join(" ")]) + // The VolumeMount for the secret operator key store certificates + .add_volume_mounts([ + VolumeMount { + mount_path: RUN_SECURITYADMIN_CERT_VOLUME_MOUNT.to_owned(), + name: RUN_SECURITYADMIN_CERT_VOLUME_NAME.to_owned(), + ..VolumeMount::default() + }, + VolumeMount { + mount_path: SECURITY_CONFIG_VOLUME_MOUNT.to_owned(), + name: SECURITY_CONFIG_VOLUME_NAME.to_owned(), + ..VolumeMount::default() + }, + ]) + .expect("the mount paths are statically defined and there should be no duplicates") + .resources( + ResourceRequirementsBuilder::new() + .with_cpu_request("100m") + .with_cpu_limit("400m") + .with_memory_request("128Mi") + .with_memory_limit("512Mi") + .build(), + ) + .build(); + + let pod_template = PodTemplateSpec { + metadata: Some(metadata.clone()), + spec: Some(PodSpec { + containers: vec![container], + + security_context: Some(PodSecurityContext { + fs_group: Some(1000), + ..PodSecurityContext::default() + }), + service_account_name: Some(self.resource_names.service_account_name()), + volumes: Some(vec![Volume { + name: SECURITY_CONFIG_VOLUME_NAME.to_owned(), + secret: Some(SecretVolumeSource { + secret_name: Some("opensearch-security-config".to_string()), + ..Default::default() + }), + ..Volume::default() + }]), + ..PodSpec::default() + }), + }; + + Job { + metadata, + spec: Some(JobSpec { + backoff_limit: Some(100), + ttl_seconds_after_finished: Some(120), + template: pod_template, + ..JobSpec::default() + }), + ..Job::default() + } + } + + fn opensearch_master_fqdn(&self) -> String { + let cluster_manager_service_name = self.resource_names.discovery_service_name(); + let namespace = &self.cluster.namespace; + let cluster_domain = &self.context_names.cluster_domain_name; + format!("{cluster_manager_service_name}.{namespace}.svc.{cluster_domain}") + } + + fn common_metadata(&self, resource_name: impl Into) -> ObjectMeta { + ObjectMetaBuilder::new() + .name(resource_name) + .namespace(&self.cluster.namespace) + .ownerreference(ownerreference_from_resource( + &self.cluster, + None, + Some(true), + )) + .with_labels(self.labels()) + .build() + } + + /// Labels on role resources + fn labels(&self) -> Labels { + // Well-known Kubernetes labels + let mut labels = Labels::role_selector( + &self.cluster, + &self.context_names.product_name.to_label_value(), + &ValidatedCluster::role_name().to_label_value(), + ) + .unwrap(); + + let managed_by = Label::managed_by( + &self.context_names.operator_name.to_string(), + &self.context_names.controller_name.to_string(), + ) + .unwrap(); + let version = Label::version(&self.cluster.product_version.to_string()).unwrap(); + + labels.insert(managed_by); + labels.insert(version); + + // Stackable-specific labels + labels + .parse_insert((STACKABLE_VENDOR_KEY, STACKABLE_VENDOR_VALUE)) + .unwrap(); + + labels + } +} From c489b44d089ac47266552edf4a69f7f89e1b414e Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 15 Aug 2025 15:48:04 +0200 Subject: [PATCH 2/6] configure tls on run-securityadmin --- rust/operator-binary/src/controller/build.rs | 3 +- .../src/controller/build/job_builder.rs | 45 ++++++++++++------- rust/operator-binary/src/framework/builder.rs | 1 + .../src/framework/builder/volume.rs | 34 ++++++++++++++ 4 files changed, 66 insertions(+), 17 deletions(-) create mode 100644 rust/operator-binary/src/framework/builder/volume.rs diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs index 051f33d..a3683bb 100644 --- a/rust/operator-binary/src/controller/build.rs +++ b/rust/operator-binary/src/controller/build.rs @@ -2,9 +2,8 @@ use std::marker::PhantomData; use role_builder::RoleBuilder; -use crate::controller::build::job_builder::JobBuilder; - use super::{ContextNames, KubernetesResources, Prepared, ValidatedCluster}; +use crate::controller::build::job_builder::JobBuilder; pub mod job_builder; pub mod node_config; diff --git a/rust/operator-binary/src/controller/build/job_builder.rs b/rust/operator-binary/src/controller/build/job_builder.rs index ee2beb7..dfde724 100644 --- a/rust/operator-binary/src/controller/build/job_builder.rs +++ b/rust/operator-binary/src/controller/build/job_builder.rs @@ -1,7 +1,10 @@ use stackable_operator::{ builder::{ meta::ObjectMetaBuilder, - pod::{container::ContainerBuilder, resources::ResourceRequirementsBuilder}, + pod::{ + container::ContainerBuilder, resources::ResourceRequirementsBuilder, + volume::SecretFormat, + }, }, k8s_openapi::api::{ batch::v1::{Job, JobSpec}, @@ -14,17 +17,20 @@ use stackable_operator::{ Label, Labels, consts::{STACKABLE_VENDOR_KEY, STACKABLE_VENDOR_VALUE}, }, + time::Duration, }; use crate::{ controller::{ContextNames, ValidatedCluster}, framework::{ - IsLabelValue, builder::meta::ownerreference_from_resource, role_utils::ResourceNames, + IsLabelValue, + builder::{meta::ownerreference_from_resource, volume::build_tls_volume}, + role_utils::ResourceNames, }, }; const RUN_SECURITYADMIN_CERT_VOLUME_NAME: &str = "tls"; -const RUN_SECURITYADMIN_CERT_VOLUME_MOUNT: &str = "/stackable/cert"; +const RUN_SECURITYADMIN_CERT_VOLUME_MOUNT: &str = "/stackable/tls-client"; const SECURITY_CONFIG_VOLUME_NAME: &str = "security-config"; const SECURITY_CONFIG_VOLUME_MOUNT: &str = "/stackable/opensearch/config/opensearch-security"; const RUN_SECURITYADMIN_CONTAINER_NAME: &str = "run-securityadmin"; @@ -61,11 +67,11 @@ impl<'a> JobBuilder<'a> { let args = [ "plugins/opensearch-security/tools/securityadmin.sh".to_string(), "-cacert".to_string(), - "config/tls-client/ca.crt".to_string(), + "/stackable/tls-client/ca.crt".to_string(), "-cert".to_string(), - "config/tls-client/tls.crt".to_string(), + "/stackable/tls-client/tls.crt".to_string(), "-key".to_string(), - "config/tls-client/tls.key".to_string(), + "/stackable/tls-client/tls.key".to_string(), "--hostname".to_string(), self.opensearch_master_fqdn(), "--configdir".to_string(), @@ -105,20 +111,29 @@ impl<'a> JobBuilder<'a> { metadata: Some(metadata.clone()), spec: Some(PodSpec { containers: vec![container], - security_context: Some(PodSecurityContext { fs_group: Some(1000), ..PodSecurityContext::default() }), + restart_policy: Some("OnFailure".to_string()), service_account_name: Some(self.resource_names.service_account_name()), - volumes: Some(vec![Volume { - name: SECURITY_CONFIG_VOLUME_NAME.to_owned(), - secret: Some(SecretVolumeSource { - secret_name: Some("opensearch-security-config".to_string()), - ..Default::default() - }), - ..Volume::default() - }]), + volumes: Some(vec![ + Volume { + name: SECURITY_CONFIG_VOLUME_NAME.to_owned(), + secret: Some(SecretVolumeSource { + secret_name: Some("opensearch-security-config".to_string()), + ..Default::default() + }), + ..Volume::default() + }, + build_tls_volume( + RUN_SECURITYADMIN_CERT_VOLUME_NAME, + Vec::::new(), + SecretFormat::TlsPem, + &Duration::from_days_unchecked(15), + None, + ), + ]), ..PodSpec::default() }), }; diff --git a/rust/operator-binary/src/framework/builder.rs b/rust/operator-binary/src/framework/builder.rs index 40caba1..078acfa 100644 --- a/rust/operator-binary/src/framework/builder.rs +++ b/rust/operator-binary/src/framework/builder.rs @@ -1,3 +1,4 @@ pub mod meta; pub mod pdb; pub mod pod; +pub mod volume; diff --git a/rust/operator-binary/src/framework/builder/volume.rs b/rust/operator-binary/src/framework/builder/volume.rs new file mode 100644 index 0000000..173eeb5 --- /dev/null +++ b/rust/operator-binary/src/framework/builder/volume.rs @@ -0,0 +1,34 @@ +use stackable_operator::{ + builder::pod::volume::{SecretFormat, SecretOperatorVolumeSourceBuilder, VolumeBuilder}, + k8s_openapi::api::core::v1::Volume, + time::Duration, +}; + +pub fn build_tls_volume( + volume_name: &str, + service_scopes: impl IntoIterator>, + secret_format: SecretFormat, + requested_secret_lifetime: &Duration, + listener_scope: Option<&str>, +) -> Volume { + let mut secret_volume_source_builder = + SecretOperatorVolumeSourceBuilder::new("tls".to_string()); + + for scope in service_scopes { + secret_volume_source_builder.with_service_scope(scope.as_ref()); + } + if let Some(listener_scope) = listener_scope { + secret_volume_source_builder.with_listener_volume_scope(listener_scope); + } + + VolumeBuilder::new(volume_name) + .ephemeral( + secret_volume_source_builder + .with_pod_scope() + .with_format(secret_format) + .with_auto_tls_cert_lifetime(*requested_secret_lifetime) + .build() + .expect("volume should be built"), + ) + .build() +} From ffa0585186c8cf1ebe342e9f4c001928b8146b3a Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 19 Aug 2025 14:47:53 +0200 Subject: [PATCH 3/6] add tls secret class to crd --- .../helm/opensearch-operator/crds/crds.yaml | 13 ++++++++ rust/operator-binary/src/controller.rs | 3 +- .../src/controller/build/job_builder.rs | 21 ++++++------ .../src/controller/build/node_config.rs | 10 +++++- .../src/controller/validate.rs | 24 ++++++++++++-- rust/operator-binary/src/crd/mod.rs | 33 +++++++++++++++++++ rust/operator-binary/src/framework.rs | 9 ++++- .../src/framework/builder/volume.rs | 5 +-- 8 files changed, 101 insertions(+), 17 deletions(-) diff --git a/deploy/helm/opensearch-operator/crds/crds.yaml b/deploy/helm/opensearch-operator/crds/crds.yaml index 7b9d514..b39df8e 100644 --- a/deploy/helm/opensearch-operator/crds/crds.yaml +++ b/deploy/helm/opensearch-operator/crds/crds.yaml @@ -25,6 +25,18 @@ spec: spec: description: A OpenSearch cluster stacklet. This resource is managed by the Stackable operator for OpenSearch. Find more information on how to use it and the resources that the operator generates in the [operator documentation](https://docs.stackable.tech/home/nightly/opensearch/). properties: + clusterConfig: + properties: + tls: + properties: + secretClass: + default: tls + description: 'Only affects client connections. This setting controls: - If TLS encryption is used at all - Which cert the servers should use to authenticate themselves against the client' + type: string + type: object + required: + - tls + type: object clusterOperation: default: reconciliationPaused: false @@ -461,6 +473,7 @@ spec: - roleGroups type: object required: + - clusterConfig - image - nodes type: object diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 35fb559..4c0303a 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -28,7 +28,7 @@ use validate::validate; use crate::{ crd::{ NodeRoles, - v1alpha1::{self}, + v1alpha1::{self, OpenSearchClusterConfig}, }, framework::{ ClusterName, ControllerName, HasNamespace, HasObjectName, HasUid, IsLabelValue, @@ -132,6 +132,7 @@ pub struct ValidatedCluster { pub name: ClusterName, pub namespace: String, pub uid: String, + pub cluster_config: OpenSearchClusterConfig, pub role_config: GenericRoleConfig, // "validated" means that labels are valid and no ugly rolegroup name broke them pub role_group_configs: BTreeMap, diff --git a/rust/operator-binary/src/controller/build/job_builder.rs b/rust/operator-binary/src/controller/build/job_builder.rs index dfde724..7cabf28 100644 --- a/rust/operator-binary/src/controller/build/job_builder.rs +++ b/rust/operator-binary/src/controller/build/job_builder.rs @@ -66,17 +66,18 @@ impl<'a> JobBuilder<'a> { let args = [ "plugins/opensearch-security/tools/securityadmin.sh".to_string(), + "--hostname".to_string(), + self.opensearch_master_fqdn(), + "--configdir".to_string(), + "config/opensearch-security/".to_string(), "-cacert".to_string(), "/stackable/tls-client/ca.crt".to_string(), "-cert".to_string(), "/stackable/tls-client/tls.crt".to_string(), "-key".to_string(), "/stackable/tls-client/tls.key".to_string(), - "--hostname".to_string(), - self.opensearch_master_fqdn(), - "--configdir".to_string(), - "config/opensearch-security/".to_string(), ]; + let mut cb = ContainerBuilder::new(RUN_SECURITYADMIN_CONTAINER_NAME) .expect("should be a valid container name"); let container = cb @@ -84,15 +85,15 @@ impl<'a> JobBuilder<'a> { .command(vec!["sh".to_string(), "-c".to_string()]) .args(vec![args.join(" ")]) // The VolumeMount for the secret operator key store certificates - .add_volume_mounts([ + .add_volume_mounts(vec![ VolumeMount { - mount_path: RUN_SECURITYADMIN_CERT_VOLUME_MOUNT.to_owned(), - name: RUN_SECURITYADMIN_CERT_VOLUME_NAME.to_owned(), + mount_path: SECURITY_CONFIG_VOLUME_MOUNT.to_owned(), + name: SECURITY_CONFIG_VOLUME_NAME.to_owned(), ..VolumeMount::default() }, VolumeMount { - mount_path: SECURITY_CONFIG_VOLUME_MOUNT.to_owned(), - name: SECURITY_CONFIG_VOLUME_NAME.to_owned(), + mount_path: RUN_SECURITYADMIN_CERT_VOLUME_MOUNT.to_owned(), + name: RUN_SECURITYADMIN_CERT_VOLUME_NAME.to_owned(), ..VolumeMount::default() }, ]) @@ -106,7 +107,6 @@ impl<'a> JobBuilder<'a> { .build(), ) .build(); - let pod_template = PodTemplateSpec { metadata: Some(metadata.clone()), spec: Some(PodSpec { @@ -128,6 +128,7 @@ impl<'a> JobBuilder<'a> { }, build_tls_volume( RUN_SECURITYADMIN_CERT_VOLUME_NAME, + &self.cluster.cluster_config.tls.secret_class, Vec::::new(), SecretFormat::TlsPem, &Duration::from_days_unchecked(15), diff --git a/rust/operator-binary/src/controller/build/node_config.rs b/rust/operator-binary/src/controller/build/node_config.rs index 6eb9ba6..046e8d3 100644 --- a/rust/operator-binary/src/controller/build/node_config.rs +++ b/rust/operator-binary/src/controller/build/node_config.rs @@ -258,7 +258,10 @@ mod tests { use super::*; use crate::{ controller::ValidatedOpenSearchConfig, - crd::NodeRoles, + crd::{ + NodeRoles, + v1alpha1::{OpenSearchClusterConfig, OpenSearchTls}, + }, framework::{ClusterName, ProductVersion, role_utils::GenericProductSpecificCommonConfig}, }; @@ -275,6 +278,11 @@ mod tests { .expect("should be a valid ClusterName"), namespace: "default".to_owned(), uid: "0b1e30e6-326e-4c1a-868d-ad6598b49e8b".to_owned(), + cluster_config: OpenSearchClusterConfig { + tls: OpenSearchTls { + secret_class: "my-tls-secret-class".to_owned(), + }, + }, role_config: GenericRoleConfig::default(), role_group_configs: BTreeMap::new(), }; diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 959ef80..ac20660 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -13,9 +13,11 @@ use super::{ ValidatedOpenSearchConfig, }; use crate::{ - crd::v1alpha1::{self, OpenSearchConfig, OpenSearchConfigFragment}, + crd::v1alpha1::{ + self, OpenSearchClusterConfig, OpenSearchConfig, OpenSearchConfigFragment, OpenSearchTls, + }, framework::{ - ClusterName, + ClusterName, TlsSecretClassName, role_utils::{GenericProductSpecificCommonConfig, RoleGroupConfig, with_validated_config}, }, }; @@ -41,6 +43,9 @@ pub enum Error { #[snafu(display("failed to set role-group name"))] ParseRoleGroupName { source: crate::framework::Error }, + #[snafu(display("failed to set tls secret class"))] + ParseTlsSecretClassName { source: crate::framework::Error }, + #[snafu(display("fragment validation failure"))] ValidateOpenSearchConfig { source: stackable_operator::config::fragment::ValidationError, @@ -70,6 +75,8 @@ pub fn validate( let product_version = ProductVersion::from_str(cluster.spec.image.product_version()) .context(ParseProductVersionSnafu)?; + let validated_cluster_config = validate_cluster_config(cluster.spec.cluster_config.clone())?; + let mut role_group_configs = BTreeMap::new(); for (raw_role_group_name, role_group_config) in &cluster.spec.nodes.role_groups { let role_group_name = @@ -88,11 +95,24 @@ pub fn validate( name: cluster_name, namespace, uid, + cluster_config: validated_cluster_config, role_config: cluster.spec.nodes.role_config.clone(), role_group_configs, }) } +fn validate_cluster_config( + cluster_config: OpenSearchClusterConfig, +) -> Result { + validate_tls_config(&cluster_config.tls)?; + Ok(cluster_config) +} + +fn validate_tls_config(tls_config: &OpenSearchTls) -> Result<()> { + TlsSecretClassName::from_str(&tls_config.secret_class).context(ParseTlsSecretClassNameSnafu)?; + Ok(()) +} + fn validate_role_group_config( context_names: &ContextNames, cluster_name: &ClusterName, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 84a6efa..c31636d 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -31,6 +31,7 @@ use crate::framework::{ }; const DEFAULT_LISTENER_CLASS: &str = "cluster-internal"; +const TLS_DEFAULT_SECRET_CLASS: &str = "tls"; #[versioned(version(name = "v1alpha1"))] pub mod versioned { @@ -57,6 +58,9 @@ pub mod versioned { // no doc string - see ProductImage struct pub image: ProductImage, + // no doc string - see ProductImage struct + pub cluster_config: OpenSearchClusterConfig, + // no doc string - see ClusterOperation struct #[serde(default)] pub cluster_operation: ClusterOperation, @@ -66,6 +70,23 @@ pub mod versioned { Role, } + #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct OpenSearchClusterConfig { + pub tls: OpenSearchTls, + } + + #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct OpenSearchTls { + /// Only affects client connections. + /// This setting controls: + /// - If TLS encryption is used at all + /// - Which cert the servers should use to authenticate themselves against the client + #[serde(default = "tls_secret_class_default")] + pub secret_class: String, + } + // The possible node roles are by default the built-in roles and the search role, see // https://github.com/opensearch-project/OpenSearch/blob/3.0.0/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java#L609-L614. // @@ -243,6 +264,18 @@ impl v1alpha1::OpenSearchConfig { } } +impl Default for v1alpha1::OpenSearchTls { + fn default() -> Self { + v1alpha1::OpenSearchTls { + secret_class: tls_secret_class_default(), + } + } +} + +fn tls_secret_class_default() -> String { + TLS_DEFAULT_SECRET_CLASS.to_string() +} + #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] pub struct NodeRoles(Vec); diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs index 07c5699..4ee212e 100644 --- a/rust/operator-binary/src/framework.rs +++ b/rust/operator-binary/src/framework.rs @@ -38,6 +38,9 @@ pub enum Error { #[allow(dead_code)] pub const MAX_OBJECT_NAME_LENGTH: usize = 253; +#[allow(dead_code)] +pub const MAX_ANNOTATION_LENGTH: usize = 253; + /// Has a name that can be used as a DNS subdomain name as defined in RFC 1123. /// Most resource types, e.g. a Pod, require such a compliant name. pub trait HasObjectName { @@ -172,7 +175,11 @@ attributed_string_type! { is_object_name, is_valid_label_value } - +attributed_string_type! { + TlsSecretClassName, + "The TLS SecretClass name", + (max_length = MAX_ANNOTATION_LENGTH - 30) +} #[cfg(test)] mod tests { use std::str::FromStr; diff --git a/rust/operator-binary/src/framework/builder/volume.rs b/rust/operator-binary/src/framework/builder/volume.rs index 173eeb5..14eb9a0 100644 --- a/rust/operator-binary/src/framework/builder/volume.rs +++ b/rust/operator-binary/src/framework/builder/volume.rs @@ -6,13 +6,14 @@ use stackable_operator::{ pub fn build_tls_volume( volume_name: &str, + tls_secret_class_name: &str, service_scopes: impl IntoIterator>, secret_format: SecretFormat, requested_secret_lifetime: &Duration, listener_scope: Option<&str>, ) -> Volume { let mut secret_volume_source_builder = - SecretOperatorVolumeSourceBuilder::new("tls".to_string()); + SecretOperatorVolumeSourceBuilder::new(tls_secret_class_name); for scope in service_scopes { secret_volume_source_builder.with_service_scope(scope.as_ref()); @@ -28,7 +29,7 @@ pub fn build_tls_volume( .with_format(secret_format) .with_auto_tls_cert_lifetime(*requested_secret_lifetime) .build() - .expect("volume should be built"), + .expect("volume should be built without parse errors"), ) .build() } From 4d7496c92516eb3251c68e98d56b02e02e4a20b8 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 20 Aug 2025 14:35:24 +0200 Subject: [PATCH 4/6] add tls volume to sts --- .../controller/build/role_group_builder.rs | 46 +++++++++++++++---- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/rust/operator-binary/src/controller/build/role_group_builder.rs b/rust/operator-binary/src/controller/build/role_group_builder.rs index 49d5f22..7263343 100644 --- a/rust/operator-binary/src/controller/build/role_group_builder.rs +++ b/rust/operator-binary/src/controller/build/role_group_builder.rs @@ -1,5 +1,8 @@ use stackable_operator::{ - builder::{meta::ObjectMetaBuilder, pod::container::ContainerBuilder}, + builder::{ + meta::ObjectMetaBuilder, + pod::{container::ContainerBuilder, volume::SecretFormat}, + }, crd::listener::{self}, k8s_openapi::{ DeepMerge, @@ -15,6 +18,7 @@ use stackable_operator::{ }, kube::api::ObjectMeta, kvp::{Label, Labels}, + time::Duration, }; use super::node_config::{CONFIGURATION_FILE_OPENSEARCH_YML, NodeConfig}; @@ -23,7 +27,7 @@ use crate::{ crd::v1alpha1, framework::{ RoleGroupName, - builder::meta::ownerreference_from_resource, + builder::{meta::ownerreference_from_resource, volume::build_tls_volume}, kvp::label::{recommended_labels, role_group_selector, role_selector}, listener::listener_pvc, role_group_utils::ResourceNames, @@ -40,6 +44,8 @@ const DATA_VOLUME_NAME: &str = "data"; const LISTENER_VOLUME_NAME: &str = "listener"; const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; +const TLS_VOLUME_NAME: &str = "tls"; +const TLS_VOLUME_DIR: &str = "/stackable/tls"; const DEFAULT_OPENSEARCH_HOME: &str = "/stackable/opensearch"; @@ -152,8 +158,13 @@ impl<'a> RoleGroupBuilder<'a> { fn build_pod_template(&self) -> PodTemplateSpec { let mut node_role_labels = Labels::new(); + let mut service_scopes = vec![self.resource_names.headless_service_name()]; + for node_role in self.role_group_config.config.node_roles.iter() { node_role_labels.insert(Self::build_node_role_label(node_role)); + if let v1alpha1::NodeRole::ClusterManager = node_role { + service_scopes.push(self.cluster.name.to_string()) + } } let metadata = ObjectMetaBuilder::new() @@ -198,14 +209,24 @@ impl<'a> RoleGroupBuilder<'a> { .config .termination_grace_period_seconds, ), - volumes: Some(vec![Volume { - name: CONFIG_VOLUME_NAME.to_owned(), - config_map: Some(ConfigMapVolumeSource { - name: self.resource_names.role_group_config_map(), - ..Default::default() - }), - ..Volume::default() - }]), + volumes: Some(vec![ + Volume { + name: CONFIG_VOLUME_NAME.to_owned(), + config_map: Some(ConfigMapVolumeSource { + name: self.resource_names.role_group_config_map(), + ..Default::default() + }), + ..Volume::default() + }, + build_tls_volume( + TLS_VOLUME_NAME, + &self.cluster.cluster_config.tls.secret_class, + service_scopes, + SecretFormat::TlsPem, + &Duration::from_days_unchecked(15), + Some(LISTENER_VOLUME_NAME), + ), + ]), ..PodSpec::default() }), }; @@ -312,6 +333,11 @@ impl<'a> RoleGroupBuilder<'a> { name: LISTENER_VOLUME_NAME.to_owned(), ..VolumeMount::default() }, + VolumeMount { + mount_path: TLS_VOLUME_DIR.to_owned(), + name: TLS_VOLUME_NAME.to_owned(), + ..VolumeMount::default() + }, ]) .expect("The mount paths are statically defined and there should be no duplicates.") .add_container_ports(vec![ From 7f805b8b000a56aa1350561b5c08c1f63e098c09 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 21 Aug 2025 10:45:35 +0200 Subject: [PATCH 5/6] add tls config to opensearch.yml --- .../src/controller/build/node_config.rs | 77 +++++++++++++++++-- .../controller/build/role_group_builder.rs | 7 +- 2 files changed, 78 insertions(+), 6 deletions(-) diff --git a/rust/operator-binary/src/controller/build/node_config.rs b/rust/operator-binary/src/controller/build/node_config.rs index 046e8d3..74d2af7 100644 --- a/rust/operator-binary/src/controller/build/node_config.rs +++ b/rust/operator-binary/src/controller/build/node_config.rs @@ -1,4 +1,4 @@ -use serde_json::{Value, json}; +use serde_json::{Map, Value, json}; use stackable_operator::builder::pod::container::FieldPathEnvVar; use super::ValidatedCluster; @@ -66,6 +66,32 @@ pub const CONFIG_OPTION_NODE_ROLES: &str = "node.roles"; /// type: list of strings pub const CONFIG_OPTION_PLUGINS_SECURITY_NODES_DN: &str = "plugins.security.nodes_dn"; +/// type: string +pub const TLS_HTTP_ENABLED: &str = "plugins.security.ssl.http.enabled"; + +/// type: string +pub const TLS_HTTP_PEMCERT_FILEPATH: &str = "plugins.security.ssl.http.pemcert_filepath"; + +/// type: string +pub const TLS_HTTP_PEMKEY_FILEPATH: &str = "plugins.security.ssl.http.pemkey_filepath"; + +/// type: string +pub const TLS_HTTP_PEMTRUSTEDCAS_FILEPATH: &str = + "plugins.security.ssl.http.pemtrustedcas_filepath"; + +/// type: string +pub const TLS_TRANSPORT_ENABLED: &str = "plugins.security.ssl.transport.enabled"; + +/// type: string +pub const TLS_TRANSPORT_PEMCERT_FILEPATH: &str = "plugins.security.ssl.transport.pemcert_filepath"; + +/// type: string +pub const TLS_TRANSPORT_PEMKEY_FILEPATH: &str = "plugins.security.ssl.transport.pemkey_filepath"; + +/// type: string +pub const TLS_TRANSPORT_PEMTRUSTEDCAS_FILEPATH: &str = + "plugins.security.ssl.transport.pemtrustedcas_filepath"; + pub struct NodeConfig { cluster: ValidatedCluster, role_group_config: OpenSearchRoleGroupConfig, @@ -88,9 +114,8 @@ impl NodeConfig { } /// static for the cluster - pub fn static_opensearch_config(&self) -> String { - let mut config = serde_json::Map::new(); - + pub fn static_opensearch_config(&self) -> Map { + let mut config = Map::new(); config.insert( CONFIG_OPTION_CLUSTER_NAME.to_owned(), json!(self.cluster.name.to_string()), @@ -111,10 +136,52 @@ impl NodeConfig { json!(["CN=generated certificate for pod".to_owned()]), ); + config + } + + pub fn tls_config(&self) -> serde_json::Map { + let mut config = Map::new(); + // TLS config for HTTP port + config.insert(TLS_HTTP_ENABLED.to_owned(), json!("true".to_string())); + config.insert( + TLS_HTTP_PEMCERT_FILEPATH.to_owned(), + json!("/stackable/tls/tls.crt".to_string()), + ); + config.insert( + TLS_HTTP_PEMKEY_FILEPATH.to_owned(), + json!("/stackable/tls/tls.key".to_string()), + ); + config.insert( + TLS_HTTP_PEMTRUSTEDCAS_FILEPATH.to_owned(), + json!("/stackable/tls/ca.crt".to_string()), + ); + // TLS config for TRANSPORT port + config.insert(TLS_TRANSPORT_ENABLED.to_owned(), json!("true".to_string())); + config.insert( + TLS_TRANSPORT_PEMCERT_FILEPATH.to_owned(), + json!("/stackable/tls/tls.crt".to_string()), + ); + config.insert( + TLS_TRANSPORT_PEMKEY_FILEPATH.to_owned(), + json!("/stackable/tls/tls.key".to_string()), + ); + config.insert( + TLS_TRANSPORT_PEMTRUSTEDCAS_FILEPATH.to_owned(), + json!("/stackable/tls/ca.crt".to_string()), + ); + + config + } + + pub fn build_config_file( + &self, + file_name: &str, + mut config: serde_json::Map, + ) -> String { for (setting, value) in self .role_group_config .config_overrides - .get(CONFIGURATION_FILE_OPENSEARCH_YML) + .get(file_name) .into_iter() .flatten() { diff --git a/rust/operator-binary/src/controller/build/role_group_builder.rs b/rust/operator-binary/src/controller/build/role_group_builder.rs index 7263343..c69fce4 100644 --- a/rust/operator-binary/src/controller/build/role_group_builder.rs +++ b/rust/operator-binary/src/controller/build/role_group_builder.rs @@ -1,3 +1,4 @@ +use serde_json::Map; use stackable_operator::{ builder::{ meta::ObjectMetaBuilder, @@ -91,9 +92,13 @@ impl<'a> RoleGroupBuilder<'a> { let metadata = self.common_metadata(self.resource_names.role_group_config_map(), Labels::new()); + let mut opensearch_yml = Map::new(); + opensearch_yml.append(&mut self.node_config.static_opensearch_config()); + opensearch_yml.append(&mut self.node_config.tls_config()); let data = [( CONFIGURATION_FILE_OPENSEARCH_YML.to_owned(), - self.node_config.static_opensearch_config(), + self.node_config + .build_config_file(CONFIGURATION_FILE_OPENSEARCH_YML, opensearch_yml), )] .into(); From c6db525100cf43031d48bf1313474542780287fa Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 22 Aug 2025 10:54:54 +0200 Subject: [PATCH 6/6] disable security demo install --- .../src/controller/build/node_config.rs | 56 ++++++++++++++++++- .../controller/build/role_group_builder.rs | 1 + 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/rust/operator-binary/src/controller/build/node_config.rs b/rust/operator-binary/src/controller/build/node_config.rs index 74d2af7..e976b6d 100644 --- a/rust/operator-binary/src/controller/build/node_config.rs +++ b/rust/operator-binary/src/controller/build/node_config.rs @@ -66,6 +66,28 @@ pub const CONFIG_OPTION_NODE_ROLES: &str = "node.roles"; /// type: list of strings pub const CONFIG_OPTION_PLUGINS_SECURITY_NODES_DN: &str = "plugins.security.nodes_dn"; +/// type: string +pub const CONFIG_OPTION_PLUGINS_SECURITY_DISABLE_INSTALL_DEMO: &str = "DISABLE_INSTALL_DEMO_CONFIG"; + +/// type: string +pub const CONFIG_OPTION_PLUGINS_SECURITY_AUDIT_TYPE: &str = "plugins.security.audit.type"; + +/// type: string +pub const CONFIG_OPTION_PLUGINS_SECURITY_AUDIT_LOG4J_LEVEL: &str = + "plugins.security.audit.config.log4j.level"; + +/// type: string +pub const CONFIG_OPTION_PLUGINS_SECURITY_LOG4J_LOGGER_NAME: &str = + "plugins.security.audit.config.log4j.logger_name"; + +/// type: string +pub const CONFIG_OPTION_PLUGINS_SECURITY_RESTAPI_ROLES_ENABLED: &str = + "plugins.security.restapi.roles_enabled"; + +/// type: string +pub const CONFIG_OPTION_PLUGINS_SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX: &str = + "allow_default_init_securityindex"; + /// type: string pub const TLS_HTTP_ENABLED: &str = "plugins.security.ssl.http.enabled"; @@ -139,7 +161,7 @@ impl NodeConfig { config } - pub fn tls_config(&self) -> serde_json::Map { + pub fn tls_config(&self) -> Map { let mut config = Map::new(); // TLS config for HTTP port config.insert(TLS_HTTP_ENABLED.to_owned(), json!("true".to_string())); @@ -173,6 +195,32 @@ impl NodeConfig { config } + pub fn security_config(&self) -> Map { + let mut config = Map::new(); + config.insert( + CONFIG_OPTION_PLUGINS_SECURITY_AUDIT_TYPE.to_owned(), + json!("log4j".to_string()), + ); + config.insert( + CONFIG_OPTION_PLUGINS_SECURITY_AUDIT_LOG4J_LEVEL.to_owned(), + json!("INFO".to_string()), + ); + config.insert( + CONFIG_OPTION_PLUGINS_SECURITY_LOG4J_LOGGER_NAME.to_owned(), + json!("oseaudit".to_string()), + ); + config.insert( + CONFIG_OPTION_PLUGINS_SECURITY_RESTAPI_ROLES_ENABLED.to_owned(), + json!("all_access".to_string()), + ); + config.insert( + CONFIG_OPTION_PLUGINS_SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX.to_owned(), + json!("true".to_string()), + ); + + config + } + pub fn build_config_file( &self, file_name: &str, @@ -208,6 +256,7 @@ impl NodeConfig { CONFIG_OPTION_INITIAL_CLUSTER_MANAGER_NODES, self.initial_cluster_manager_nodes(), ) + .with_value(CONFIG_OPTION_PLUGINS_SECURITY_DISABLE_INSTALL_DEMO, "true") .with_value( CONFIG_OPTION_NODE_ROLES, self.role_group_config @@ -381,6 +430,11 @@ mod tests { // TODO Test EnvVarSet and compare EnvVarSets assert_eq!( vec![ + EnvVar { + name: "DISABLE_INSTALL_DEMO_CONFIG".to_owned(), + value: Some("true".to_owned()), + value_from: None + }, EnvVar { name: "TEST".to_owned(), value: Some("value".to_owned()), diff --git a/rust/operator-binary/src/controller/build/role_group_builder.rs b/rust/operator-binary/src/controller/build/role_group_builder.rs index c69fce4..07a8a21 100644 --- a/rust/operator-binary/src/controller/build/role_group_builder.rs +++ b/rust/operator-binary/src/controller/build/role_group_builder.rs @@ -95,6 +95,7 @@ impl<'a> RoleGroupBuilder<'a> { let mut opensearch_yml = Map::new(); opensearch_yml.append(&mut self.node_config.static_opensearch_config()); opensearch_yml.append(&mut self.node_config.tls_config()); + opensearch_yml.append(&mut self.node_config.security_config()); let data = [( CONFIGURATION_FILE_OPENSEARCH_YML.to_owned(), self.node_config