From 610b84d027d16d9562148c327f168e5b8d358d98 Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Wed, 30 Aug 2023 17:09:48 +0530 Subject: [PATCH 1/6] Add new service ceilometer-agent-compute Add new service to run ceilometer agent polling for compute node. Add ceilometer configuration and polling template files. Update nova configuration template to include oslo notification messaging driver. Add config to enable/disable the service and telemetry publish secret key. --- openstack_hypervisor/api.py | 7 +++++ openstack_hypervisor/hooks.py | 29 ++++++++++++++++++ openstack_hypervisor/model.py | 7 +++++ openstack_hypervisor/services.py | 17 +++++++++++ setup.cfg | 1 + snap/snapcraft.yaml | 11 +++++++ templates/ceilometer.conf.j2 | 52 ++++++++++++++++++++++++++++++++ templates/nova.conf.j2 | 5 +++ templates/polling.yaml.j2 | 19 ++++++++++++ tests/unit/test_hooks.py | 11 +++++++ 10 files changed, 159 insertions(+) create mode 100644 templates/ceilometer.conf.j2 create mode 100644 templates/polling.yaml.j2 diff --git a/openstack_hypervisor/api.py b/openstack_hypervisor/api.py index b7d32a7..d4f2bd8 100644 --- a/openstack_hypervisor/api.py +++ b/openstack_hypervisor/api.py @@ -31,6 +31,7 @@ "network": model.NetworkConfig, "node": model.NodeConfig, "logging": model.LoggingConfig, + "telemetry": model.TelemetryConfig, } @@ -138,6 +139,12 @@ async def update_logging(config: model.LoggingConfig): return _update_settings("logging", config) +@app.patch("/settings/telemetry") +async def update_telemetry(config: model.TelemetryConfig): + """Updates telemetry section settings.""" + return _update_settings("telemetry", config) + + @app.post("/reset") async def reset_config(): """Reset all configs to default.""" diff --git a/openstack_hypervisor/hooks.py b/openstack_hypervisor/hooks.py index 0e51bb6..92034d9 100644 --- a/openstack_hypervisor/hooks.py +++ b/openstack_hypervisor/hooks.py @@ -70,6 +70,7 @@ Path("etc/neutron/neutron.conf.d"), Path("etc/ssl/certs"), Path("etc/ssl/private"), + Path("etc/ceilometer"), # log Path("log/libvirt/qemu"), Path("log/ovn"), @@ -241,6 +242,9 @@ def _get_local_ip_by_default_route() -> str: "node.fqdn": socket.getfqdn, "node.ip-address": _get_local_ip_by_default_route, # noqa: F821 # TLS + # Telemetry + "telemetry.enable": False, + "telemetry.publisher-secret": UNSET, } @@ -256,6 +260,12 @@ def _get_local_ip_by_default_route() -> str: "network", ], "neutron-ovn-metadata-agent": ["credentials", "network", "node", "network.ovn_key"], + "ceilometer-compute-agent": [ + "identity.password", + "identity.username", + "identity", + "rabbitmq.url", + ], } @@ -335,6 +345,14 @@ def _context_compat(context: Dict[str, Any]) -> Dict[str, Any]: Path("etc/openvswitch/system-id.conf"): { "template": "system-id.conf.j2", }, + Path("etc/ceilometer/ceilometer.conf"): { + "template": "ceilometer.conf.j2", + "services": ["ceilometer-compute-agent"], + }, + Path("etc/ceilometer/polling.yaml"): { + "template": "polling.yaml.j2", + "services": ["ceilometer-compute-agent"], + }, } @@ -958,6 +976,15 @@ def _services_not_ready(context: dict) -> List[str]: return sorted(list(set(not_ready))) +def _services_not_enabled_by_config(context: dict) -> List[str]: + """Check if services are enabled by configuration.""" + not_enabled = [] + if not context.get("telemetry", {}).get("enable"): + not_enabled.append("ceilometer-compute-agent") + + return not_enabled + + def configure(snap: Snap) -> None: """Runs the `configure` hook for the snap. @@ -984,6 +1011,7 @@ def configure(snap: Snap) -> None: "node", "rabbitmq", "credentials", + "telemetry", ).as_dict() # Add some general snap path information @@ -997,6 +1025,7 @@ def configure(snap: Snap) -> None: context = _context_compat(context) logging.info(context) exclude_services = _services_not_ready(context) + exclude_services.extend(_services_not_enabled_by_config(context)) logging.warning(f"{exclude_services} are missing required config, stopping") services = snap.services.list() for service in exclude_services: diff --git a/openstack_hypervisor/model.py b/openstack_hypervisor/model.py index bfc43ec..254965b 100644 --- a/openstack_hypervisor/model.py +++ b/openstack_hypervisor/model.py @@ -103,3 +103,10 @@ class LoggingConfig(BaseModel): """Data model for the logging configuration for the hypervisor.""" debug: bool = Field(default=False) + + +class TelemetryConfig(BaseModel): + """Data model for telemetry configuration settings.""" + + enable: bool = Field(default=False) + publisher_secret: Optional[str] = Field(alias="publisher-secret") diff --git a/openstack_hypervisor/services.py b/openstack_hypervisor/services.py index c729586..298bae0 100644 --- a/openstack_hypervisor/services.py +++ b/openstack_hypervisor/services.py @@ -35,6 +35,7 @@ class OpenStackService: conf_files = [] conf_dirs = [] + extra_args = [] executable = None @@ -70,6 +71,7 @@ def run(self, snap: Snap) -> int: cmd = [str(executable)] cmd.extend(args) + cmd.extend(self.extra_args) completed_process = subprocess.run(cmd) logging.info(f"Exiting with code {completed_process.returncode}") @@ -125,6 +127,21 @@ class NeutronOVNMetadataAgentService(OpenStackService): neutron_ovn_metadata_agent = partial(entry_point, NeutronOVNMetadataAgentService) +class CeilometerComputeAgentService(OpenStackService): + """A python service object used to run the ceilometer-agent-compute daemon.""" + + conf_files = [ + Path("etc/ceilometer/ceilometer.conf"), + ] + conf_dirs = [] + extra_args = ["--polling-namespaces", "compute"] + + executable = Path("usr/bin/ceilometer-polling") + + +ceilometer_compute_agent = partial(entry_point, CeilometerComputeAgentService) + + class OVSDBServerService: """A python service object used to run the ovsdb-server daemon.""" diff --git a/setup.cfg b/setup.cfg index ad97be8..bfe9aa4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,6 +30,7 @@ console_scripts = nova-api-metadata-service = openstack_hypervisor.services:nova_api_metadata ovsdb-server-service = openstack_hypervisor.services:ovsdb_server neutron-ovn-metadata-agent-service = openstack_hypervisor.services:neutron_ovn_metadata_agent + ceilometer-compute-agent-service = openstack_hypervisor.services:ceilometer_compute_agent snaphelpers.hooks = configure = openstack_hypervisor.hooks:configure diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 76e4c2d..f93ea08 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -233,6 +233,16 @@ apps: - firewall-control - microstack-support + ceilometer-compute-agent: + command: 'bin/ceilometer-compute-agent-service' + after: [libvirtd] + daemon: simple + plugs: + - network + - network-bind + - firewall-control + - microstack-support + hypervisor-config-service: command: 'bin/gunicorn openstack_hypervisor.api:app --workers 1 --worker-class uvicorn.workers.UvicornWorker --timeout 120 --bind unix:$SNAP_DATA/run/hypervisor-config/unix.socket' restart-condition: on-failure @@ -540,6 +550,7 @@ parts: openstack: plugin: nil stage-packages: + - ceilometer-agent-compute - python3-nova - python3-neutron - python3-os-vif diff --git a/templates/ceilometer.conf.j2 b/templates/ceilometer.conf.j2 new file mode 100644 index 0000000..dd3095a --- /dev/null +++ b/templates/ceilometer.conf.j2 @@ -0,0 +1,52 @@ +# THIS FILE IS MANAGED BY THE SNAP - CHANGES WILL BE OVERWRITTEN +# Use $SNAP_COMMON/etc/ceilometer/ceilometer.conf.d for deployment specific +# configuration +[DEFAULT] +debug = {{ logging.debug }} + +# AMQP connection to RabbitMQ +transport_url = {{ rabbitmq.url }} + +[polling] +batch_size = 50 +cfg_file = {{ snap_common }}/etc/ceilometer/polling.yaml +pollsters_definitions_dirs = {{ snap_common }}/etc/ceilometer/pollsters.d + +{% if telemetry.publisher_secret -%} +[publisher] +telemetry_secret = {{ telemetry.publisher_secret }} +{%- endif %} + +[gnocchi] +filter_service_activity = False +archive_policy = low + +[keystone_authtoken] +auth_url = {{ identity.auth_url }} +auth_type = password +project_domain_name = {{ identity.project_domain_name }} +user_domain_name = {{ identity.user_domain_name }} +project_name = {{ identity.project_name }} +username = {{ identity.username }} +password = {{ identity.password }} +service_token_roles = {{ identity.admin_role }} +service_token_roles_required = True + +[service_user] +send_service_user_token = true +auth_type = password +auth_url = {{ identity.auth_url }} +project_domain_id = {{ identity.project_domain_id }} +user_domain_id = {{ identity.user_domain_id }} +project_name = {{ identity.project_name }} +username = {{ identity.username }} +password = {{ identity.password }} + +[service_credentials] +auth_type = password +auth_url = {{ identity.auth_url }} +project_domain_id = {{ identity.project_domain_id }} +user_domain_id = {{ identity.user_domain_id }} +project_name = {{ identity.project_name }} +username = {{ identity.username }} +password = {{ identity.password }} diff --git a/templates/nova.conf.j2 b/templates/nova.conf.j2 index 726f2db..51620e3 100644 --- a/templates/nova.conf.j2 +++ b/templates/nova.conf.j2 @@ -108,3 +108,8 @@ region_name = {{ identity.region_name }} project_name = {{ identity.project_name }} username = {{ identity.username }} password = {{ identity.password }} + ++{% if telemetry.enable -%} ++[oslo_messaging_notifications] ++driver = messagingv2 ++{%- endif %} diff --git a/templates/polling.yaml.j2 b/templates/polling.yaml.j2 new file mode 100644 index 0000000..0faf04b --- /dev/null +++ b/templates/polling.yaml.j2 @@ -0,0 +1,19 @@ +# THIS FILE IS MANAGED BY THE SNAP - CHANGES WILL BE OVERWRITTEN +{% if telemetry.enable -%} +--- +sources: + - name: some_pollsters + interval: 300 + meters: + - cpu + - cpu_l3_cache + - memory.usage + - network.incoming.bytes + - network.incoming.packets + - network.outgoing.bytes + - network.outgoing.packets + - disk.device.read.bytes + - disk.device.read.requests + - disk.device.write.bytes + - disk.device.write.requests +{%- endif %} diff --git a/tests/unit/test_hooks.py b/tests/unit/test_hooks.py index 593aee8..a2a346d 100644 --- a/tests/unit/test_hooks.py +++ b/tests/unit/test_hooks.py @@ -134,6 +134,7 @@ def test_configure_hook_exception(self, mocker, snap, os_makedirs, check_call): def test_services(self): """Test getting a list of managed services.""" assert hooks.services() == [ + "ceilometer-compute-agent", "libvirtd", "neutron-ovn-metadata-agent", "nova-api-metadata", @@ -162,12 +163,14 @@ def test_check_config_present(self): def test_services_not_ready(self, snap): config = {} assert hooks._services_not_ready(config) == [ + "ceilometer-compute-agent", "neutron-ovn-metadata-agent", "nova-api-metadata", "nova-compute", ] config["identity"] = {"username": "user", "password": "pass"} assert hooks._services_not_ready(config) == [ + "ceilometer-compute-agent", "neutron-ovn-metadata-agent", "nova-api-metadata", "nova-compute", @@ -188,6 +191,14 @@ def test_services_not_ready(self, snap): config["credentials"] = {"ovn_metadata_proxy_shared_secret": "secret"} assert hooks._services_not_ready(config) == [] + def test_services_not_enabled_by_config(self, snap): + config = {} + assert hooks._services_not_enabled_by_config(config) == [ + "ceilometer-compute-agent", + ] + config["telemetry"] = {"enable": True} + assert hooks._services_not_enabled_by_config(config) == [] + def test_list_bridge_ifaces(self, check_output): check_output.return_value = b"int1\nint2\n" assert hooks._list_bridge_ifaces("br1") == ["int1", "int2"] From f78d2b75db930932dbb25a692ad0e5d9dfbf686c Mon Sep 17 00:00:00 2001 From: Chi Wai Chan Date: Wed, 16 Aug 2023 18:33:01 +0800 Subject: [PATCH 2/6] Add libvirt-exporter service for exporting libvirt related metrics. --- snap/snapcraft.yaml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index f93ea08..eea5c29 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -261,6 +261,17 @@ apps: listen-stream: $SNAP_DATA/run/hypervisor-config/unix.socket socket-mode: 0666 + # Exporters + libvirt-exporter: + daemon: simple + install-mode: disable + restart-condition: on-abnormal + command: 'bin/libvirt-exporter' + plugs: + - libvirt + - network-bind + + parts: kvm-support: plugin: nil @@ -586,6 +597,26 @@ parts: craftctl default snap-helpers write-hooks + libvirt-exporter: + plugin: go + source-tag: 2.3.3 + source-type: git + source: https://github.com/Tinkoff/libvirt-exporter + build-snaps: + - go + after: + - libvirt + build-packages: + - pkg-config + - libvirt-dev + stage-packages: + - libdb5.3 + override-prime: | + craftctl default + # Drop the driver for Xen introduced by libvirt-dev, it will cause + # libvirt failed to start. + rm -rf $CRAFT_PRIME/usr/lib/libvirt/connection-driver/libvirt_driver_libxl.so + slots: hypervisor-config: interface: content From 68350f47bacf24ceadac4123a6f215b69db1a1b8 Mon Sep 17 00:00:00 2001 From: Chi Wai Chan Date: Thu, 24 Aug 2023 16:57:08 +0800 Subject: [PATCH 3/6] Add snap config option to enable or disable monitoring service. --- openstack_hypervisor/api.py | 7 +++++++ openstack_hypervisor/hooks.py | 30 +++++++++++++++++++++++++++++- openstack_hypervisor/model.py | 6 ++++++ snap/snapcraft.yaml | 2 +- 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/openstack_hypervisor/api.py b/openstack_hypervisor/api.py index d4f2bd8..6dbf493 100644 --- a/openstack_hypervisor/api.py +++ b/openstack_hypervisor/api.py @@ -32,6 +32,7 @@ "node": model.NodeConfig, "logging": model.LoggingConfig, "telemetry": model.TelemetryConfig, + "monitoring": model.MonitoringConfig, } @@ -145,6 +146,12 @@ async def update_telemetry(config: model.TelemetryConfig): return _update_settings("telemetry", config) +@app.patch("/settings/monitoring") +async def update_monitoring(config: model.MonitoringConfig): + """Updates monitoring section settings.""" + return _update_settings("monitoring", config) + + @app.post("/reset") async def reset_config(): """Reset all configs to default.""" diff --git a/openstack_hypervisor/hooks.py b/openstack_hypervisor/hooks.py index 92034d9..62e3c00 100644 --- a/openstack_hypervisor/hooks.py +++ b/openstack_hypervisor/hooks.py @@ -93,6 +93,11 @@ Path("run/hypervisor-config"), ] +# As defined in the snap/snapcraft.yaml +MONITORING_SERVICES = [ + "libvirt-exporter", +] + def _generate_secret(length: int = DEFAULT_SECRET_LENGTH) -> str: """Generate a secure secret. @@ -237,6 +242,8 @@ def _get_local_ip_by_default_route() -> str: "network.enable-gateway": False, "network.ip-address": _get_local_ip_by_default_route, # noqa: F821 "network.external-nic": UNSET, + # Monitoring + "monitoring.enable": False, # General "logging.debug": False, "node.fqdn": socket.getfqdn, @@ -917,7 +924,7 @@ def _is_hw_virt_supported() -> bool: return False -def _configure_kvm(snap) -> None: +def _configure_kvm(snap: Snap) -> None: """Configure KVM hardware virtualization. :param snap: the snap reference @@ -937,6 +944,25 @@ def _configure_kvm(snap) -> None: snap.config.set({"compute.virt-type": "qemu"}) +def _configure_monitoring_services(snap: Snap) -> None: + """Configure all the monitoring services. + + :param snap: the snap reference + :type snap: Snap + :return: None + """ + services = snap.services.list() + enable_monitoring = snap.config.get("monitoring.enable") + if enable_monitoring: + logging.info("Enabling all exporter services.") + for service in MONITORING_SERVICES: + services[service].start(enable=True) + else: + logging.info("Disabling all exporter services.") + for service in MONITORING_SERVICES: + services[service].stop(disable=True) + + def services() -> List[str]: """List of services managed by hooks.""" return sorted(list(set([w for v in TEMPLATES.values() for w in v.get("services", [])]))) @@ -1012,6 +1038,7 @@ def configure(snap: Snap) -> None: "rabbitmq", "credentials", "telemetry", + "monitoring", ).as_dict() # Add some general snap path information @@ -1050,3 +1077,4 @@ def configure(snap: Snap) -> None: _configure_ovn_external_networking(snap) _configure_ovn_tls(snap) _configure_kvm(snap) + _configure_monitoring_services(snap) diff --git a/openstack_hypervisor/model.py b/openstack_hypervisor/model.py index 254965b..04e8511 100644 --- a/openstack_hypervisor/model.py +++ b/openstack_hypervisor/model.py @@ -110,3 +110,9 @@ class TelemetryConfig(BaseModel): enable: bool = Field(default=False) publisher_secret: Optional[str] = Field(alias="publisher-secret") + + +class MonitoringConfig(BaseModel): + """Data model for the monitoring configuration settings.""" + + enable: bool = Field(default=False) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index eea5c29..28e8b3a 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -266,7 +266,7 @@ apps: daemon: simple install-mode: disable restart-condition: on-abnormal - command: 'bin/libvirt-exporter' + command: 'bin/libvirt-exporter --web.listen-address :9177 --web.telemetry-path /metrics --libvirt.uri qemu:///system' plugs: - libvirt - network-bind From 96e71dac06c14710ae1299a67e6d8f47bdb828f6 Mon Sep 17 00:00:00 2001 From: Chi Wai Chan Date: Thu, 31 Aug 2023 14:21:07 +0800 Subject: [PATCH 4/6] Optimized the build of libvirt exporter. --- snap/snapcraft.yaml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 28e8b3a..3612810 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -606,16 +606,9 @@ parts: - go after: - libvirt - build-packages: - - pkg-config - - libvirt-dev - stage-packages: - - libdb5.3 - override-prime: | - craftctl default - # Drop the driver for Xen introduced by libvirt-dev, it will cause - # libvirt failed to start. - rm -rf $CRAFT_PRIME/usr/lib/libvirt/connection-driver/libvirt_driver_libxl.so + build-environment: + - CGO_CFLAGS: "-I$CRAFT_STAGE/usr/include" + - CGO_LDFLAGS: "-L$CRAFT_STAGE/usr/lib" slots: hypervisor-config: From 67dd6e16d89274e80a7546c17334882d8d2fd52d Mon Sep 17 00:00:00 2001 From: jneo8 Date: Thu, 17 Aug 2023 15:53:08 +0545 Subject: [PATCH 5/6] feat(services): Add ovs exporter service Create a python service object to run the ovs-exporter daemon --- openstack_hypervisor/hooks.py | 1 + openstack_hypervisor/services.py | 53 ++++++++++++++++++++++++++++++++ setup.cfg | 1 + snap/snapcraft.yaml | 49 ++++++++++++++++++++++++++++- 4 files changed, 103 insertions(+), 1 deletion(-) diff --git a/openstack_hypervisor/hooks.py b/openstack_hypervisor/hooks.py index 62e3c00..ff1a261 100644 --- a/openstack_hypervisor/hooks.py +++ b/openstack_hypervisor/hooks.py @@ -96,6 +96,7 @@ # As defined in the snap/snapcraft.yaml MONITORING_SERVICES = [ "libvirt-exporter", + "ovs-exporter", ] diff --git a/openstack_hypervisor/services.py b/openstack_hypervisor/services.py index 298bae0..2b9283e 100644 --- a/openstack_hypervisor/services.py +++ b/openstack_hypervisor/services.py @@ -174,3 +174,56 @@ def run(self, snap: Snap) -> int: ovsdb_server = partial(entry_point, OVSDBServerService) + + +class OVSExporterService: + """A python service object used to run the ovs-exporter daemon.""" + + def run(self, snap: Snap) -> int: + """Runs the ovs-exporter service. + + Invoked when config monitoring is enable. + + :param snap: the snap context + :type snap: Snap + :return: exit code of the process + :rtype: int + """ + setup_logging(snap.paths.common / "ovs-exporter.log") + executable = snap.paths.snap / "bin" / "ovs-exporter" + listen_address = ":9475" + args = [ + f"-web.listen-address={listen_address}", + "-database.vswitch.file.data.path", + f"{snap.paths.common}/etc/openvswitch/conf.db", + "-database.vswitch.file.log.path", + f"{snap.paths.common}/log/openvswitch/ovsdb-server.log", + "-database.vswitch.file.pid.path", + f"{snap.paths.common}/run/openvswitch/ovsdb-server.pid", + "-database.vswitch.file.system.id.path", + f"{snap.paths.common}/etc/openvswitch/system-id.conf", + "-database.vswitch.name", + "Open_vSwitch", + "-database.vswitch.socket.remote", + "unix:" + f"{snap.paths.common}/run/openvswitch/db.sock", + "-service.ovncontroller.file.log.path", + f"{snap.paths.common}/log/ovn/ovn-controller.log", + "-service.ovncontroller.file.pid.path", + f"{snap.paths.common}/run/ovn/ovn-controller.pid", + "-service.vswitchd.file.log.path", + f"{snap.paths.common}/log/openvswitch/ovs-vswitchd.log", + "-service.vswitchd.file.pid.path", + f"{snap.paths.common}/run/openvswitch/ovs-vswitchd.pid", + "-system.run.dir", + f"{snap.paths.common}/run/openvswitch", + ] + cmd = [str(executable)] + cmd.extend(args) + + completed_process = subprocess.run(cmd) + + logging.info(f"Exiting with code {completed_process.returncode}") + return completed_process.returncode + + +ovs_exporter = partial(entry_point, OVSExporterService) diff --git a/setup.cfg b/setup.cfg index bfe9aa4..46a5d27 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,6 +31,7 @@ console_scripts = ovsdb-server-service = openstack_hypervisor.services:ovsdb_server neutron-ovn-metadata-agent-service = openstack_hypervisor.services:neutron_ovn_metadata_agent ceilometer-compute-agent-service = openstack_hypervisor.services:ceilometer_compute_agent + ovs-exporter-service = openstack_hypervisor.services:ovs_exporter snaphelpers.hooks = configure = openstack_hypervisor.hooks:configure diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 3612810..f6c6190 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -124,6 +124,7 @@ apps: - hardware-observe - hugepages-control - microstack-support + ovsdb-server: command: "bin/ovsdb-server-service" stop-command: usr/share/openvswitch/scripts/ovs-ctl --no-ovs-vswitchd stop @@ -261,7 +262,6 @@ apps: listen-stream: $SNAP_DATA/run/hypervisor-config/unix.socket socket-mode: 0666 - # Exporters libvirt-exporter: daemon: simple install-mode: disable @@ -271,6 +271,18 @@ apps: - libvirt - network-bind + ovs-exporter: + command: 'bin/ovs-exporter-service' + plugs: + - network-bind + - openvswitch + - log-observe + - system-observe + - network-observe + - netlink-audit + - kernel-module-observe + daemon: simple + install-mode: disable parts: kvm-support: @@ -279,6 +291,41 @@ parts: - on amd64: - msr-tools + ovs-exporter: + plugin: go + source: https://github.com/greenpau/ovs_exporter.git + source-tag: v1.0.7 + build-packages: + - gcc + - golang-go + override-build: | + GIT_COMMIT=$(git describe --dirty --always) + GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD -- | head -1) + BUILD_USER=snapcraft + BUILD_DATE=$(date +"%Y-%m-%d") + BINARY=ovs-exporter + VERBOSE=-v + PROJECT=ovs_exporter + PKG_DIR=pkg/ovs_exporter + mkdir -p ./bin/ + CGO_ENABLED=0 go build -o ./bin/${BINARY} \ + -ldflags="-w -s \ + -X github.com/prometheus/common/version.Version=$SNAPCRAFT_PROJECT_VERSION \ + -X github.com/prometheus/common/version.Revision=$GIT_COMMIT \ + -X github.com/prometheus/common/version.Branch=$GIT_BRANCH \ + -X github.com/prometheus/common/version.BuildUser=$BUILD_USER \ + -X github.com/prometheus/common/version.BuildDate=$BUILD_DATE \ + -X ${PROJECT}/${PKG_DIR}.appName=$BINARY \ + -X ${PROJECT}/${PKG_DIR}.appVersion=$SNAPCRAFT_PROJECT_VERSION \ + -X ${PROJECT}/${PKG_DIR}.gitBranch=$GIT_BRANCH \ + -X ${PROJECT}/${PKG_DIR}.gitCommit=$GIT_COMMIT \ + -X ${PROJECT}/${PKG_DIR}.buildUser=$BUILD_USER \ + -X ${PROJECT}/${PKG_DIR}.buildDate=$BUILD_DATE" \ + ./cmd/ovs_exporter/*.go + override-prime: | + snapcraftctl prime + install -D $SNAPCRAFT_PART_SRC/../build/bin/ovs-exporter bin/ovs-exporter + qemu: source: https://git.launchpad.net/ubuntu/+source/qemu source-type: git From 70bbaae4a3e2a4b9fdd206d439328a30f3216eec Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 5 Jun 2023 16:28:43 +0000 Subject: [PATCH 6/6] Add ceph client configuration Signed-off-by: Liam Young --- openstack_hypervisor/hooks.py | 80 +++++++++++++++++++++++++++++++++++ openstack_hypervisor/model.py | 3 ++ templates/nova.conf.j2 | 4 ++ tests/unit/test_hooks.py | 67 +++++++++++++++++++++++++++++ 4 files changed, 154 insertions(+) diff --git a/openstack_hypervisor/hooks.py b/openstack_hypervisor/hooks.py index ff1a261..9cb6225 100644 --- a/openstack_hypervisor/hooks.py +++ b/openstack_hypervisor/hooks.py @@ -92,6 +92,16 @@ Path("lib/neutron"), Path("run/hypervisor-config"), ] +SECRET_XML = string.Template( + """ + + $uuid + + client.cinder-ceph secret + + +""" +) # As defined in the snap/snapcraft.yaml MONITORING_SERVICES = [ @@ -230,6 +240,9 @@ def _get_local_ip_by_default_route() -> str: "compute.virt-type": "auto", "compute.cpu-models": UNSET, "compute.spice-proxy-address": _get_local_ip_by_default_route, # noqa: F821 + "compute.rbd_user": "nova", + "compute.rbd_secret_uuid": UNSET, + "compute.rbd_key": UNSET, # Neutron "network.physnet-name": "physnet1", "network.external-bridge": "br-ex", @@ -925,6 +938,72 @@ def _is_hw_virt_supported() -> bool: return False +def _set_secret(conn, secret_uuid: str, secret_value: str) -> None: + """Set the ceph access secret in libvirt.""" + logging.info(f"Setting secret {secret_uuid}") + new_secret = conn.secretDefineXML(SECRET_XML.substitute(uuid=secret_uuid)) + # nova assumes the secret is raw and always encodes it *1, so decode it + # before storing it. + # *1 https://opendev.org/openstack/nova/src/branch/stable/2023.1/nova/ + # virt/libvirt/imagebackend.py#L1110 + new_secret.setValue(base64.b64decode(secret_value)) + + +def _get_libvirt(): + # Lazy import libvirt otherwise snap will not build + import libvirt + + return libvirt + + +def _ensure_secret(secret_uuid: str, secret_value: str) -> None: + """Ensure libvirt has the ceph access secret with the correct value.""" + libvirt = _get_libvirt() + conn = libvirt.open("qemu:///system") + # Check if secret exists + if secret_uuid in conn.listSecrets(): + logging.info(f"Found secret {secret_uuid}") + # check secret matches + secretobj = conn.secretLookupByUUIDString(secret_uuid) + try: + secret = secretobj.value() + except libvirt.libvirtError as e: + if e.get_error_code() == libvirt.VIR_ERR_NO_SECRET: + logging.info(f"Secret {secret_uuid} has no value.") + secret = None + else: + raise + # Secret is stored raw so encode it before comparison. + if secret == base64.b64encode(secret_value.encode()): + logging.info(f"Secret {secret_uuid} has desired value.") + else: + logging.info(f"Secret {secret_uuid} has wrong value, replacing.") + secretobj.undefine() + _set_secret(conn, secret_uuid, secret_value) + else: + logging.info(f"Secret {secret_uuid} not found, creating.") + _set_secret(conn, secret_uuid, secret_value) + + +def _configure_ceph(snap) -> None: + """Configure ceph client. + + :param snap: the snap reference + :type snap: Snap + :return: None + """ + logging.info("Configuring ceph access") + context = ( + snap.config.get_options( + "compute", + ) + .as_dict() + .get("compute") + ) + if all(k in context for k in ("rbd-key", "rbd-secret-uuid")): + _ensure_secret(context["rbd-secret-uuid"], context["rbd-key"]) + + def _configure_kvm(snap: Snap) -> None: """Configure KVM hardware virtualization. @@ -1079,3 +1158,4 @@ def configure(snap: Snap) -> None: _configure_ovn_tls(snap) _configure_kvm(snap) _configure_monitoring_services(snap) + _configure_ceph(snap) diff --git a/openstack_hypervisor/model.py b/openstack_hypervisor/model.py index 04e8511..540b992 100644 --- a/openstack_hypervisor/model.py +++ b/openstack_hypervisor/model.py @@ -73,6 +73,9 @@ class ComputeConfig(BaseModel): virt_type: str = Field(alias="virt-type", default="auto") cpu_models: Optional[str] = Field(alias="cpu-models") spice_proxy_address: Optional[IPvAnyAddress] = Field(alias="spice-proxy-address") + rbd_user: Optional[str] = Field(alias="rbd-user", default="nova") + rbd_secret_uuid: Optional[str] = Field(alias="rbd-secret-uuid") + rbd_key: Optional[str] = Field(alias="rbd-key") class NetworkConfig(BaseModel): diff --git a/templates/nova.conf.j2 b/templates/nova.conf.j2 index 51620e3..e9b3e8a 100644 --- a/templates/nova.conf.j2 +++ b/templates/nova.conf.j2 @@ -33,6 +33,10 @@ cpu_mode = {{ compute.cpu_mode }} cpu_models = {{ compute.cpu_models }} {% endif %} +{% if compute.rbd_secret_uuid %} +rbd_user = {{ compute.rbd_user }} +rbd_secret_uuid = {{ compute.rbd_secret_uuid }} +{% endif %} [oslo_concurrency] # Oslo Concurrency lock path diff --git a/tests/unit/test_hooks.py b/tests/unit/test_hooks.py index a2a346d..ff03b41 100644 --- a/tests/unit/test_hooks.py +++ b/tests/unit/test_hooks.py @@ -328,3 +328,70 @@ def test_del_external_nics_from_bridge(self, mocker): hooks._del_external_nics_from_bridge("br-ex") expect = [mock.call("br-ex", "eth0"), mock.call("br-ex", "eth1")] mock_del_interface_from_bridge.assert_has_calls(expect) + + def test_set_secret(self, mocker): + conn_mock = mocker.Mock() + secret_mock = mocker.Mock() + conn_mock.secretDefineXML.return_value = secret_mock + hooks._set_secret(conn_mock, "uuid1", "c2VjcmV0Cg==") + conn_mock.secretDefineXML.assert_called_once() + secret_mock.setValue.assert_called_once_with(b"secret\n") + + def test_ensure_secret_new_secret(self, mocker): + conn_mock = mocker.Mock() + mock_libvirt = mocker.Mock() + mock_get_libvirt = mocker.patch.object(hooks, "_get_libvirt") + mock_get_libvirt.return_value = mock_libvirt + mock_libvirt.open.return_value = conn_mock + mock_set_secret = mocker.patch.object(hooks, "_set_secret") + conn_mock.listSecrets.return_value = [] + hooks._ensure_secret("uuid1", "secret") + mock_set_secret.assert_called_once_with(conn_mock, "uuid1", "secret") + + def test_ensure_secret_secret_exists(self, mocker): + conn_mock = mocker.Mock() + mock_libvirt = mocker.Mock() + secret_mock = mocker.Mock() + secret_mock.value.return_value = b"c2VjcmV0" + mock_get_libvirt = mocker.patch.object(hooks, "_get_libvirt") + mock_get_libvirt.return_value = mock_libvirt + mock_libvirt.open.return_value = conn_mock + mock_set_secret = mocker.patch.object(hooks, "_set_secret") + conn_mock.listSecrets.return_value = ["uuid1"] + conn_mock.secretLookupByUUIDString.return_value = secret_mock + hooks._ensure_secret("uuid1", "secret") + assert not mock_set_secret.called + + def test_ensure_secret_secret_wrong_value(self, mocker): + conn_mock = mocker.Mock() + mock_libvirt = mocker.Mock() + secret_mock = mocker.Mock() + secret_mock.value.return_value = b"wrong" + mock_get_libvirt = mocker.patch.object(hooks, "_get_libvirt") + mock_get_libvirt.return_value = mock_libvirt + mock_libvirt.open.return_value = conn_mock + mock_set_secret = mocker.patch.object(hooks, "_set_secret") + conn_mock.listSecrets.return_value = ["uuid1"] + conn_mock.secretLookupByUUIDString.return_value = secret_mock + hooks._ensure_secret("uuid1", "secret") + mock_set_secret.assert_called_once_with(conn_mock, "uuid1", "secret") + + def test_ensure_secret_secret_missing_value(self, mocker): + class FakeError(Exception): + def get_error_code(self): + return 42 + + conn_mock = mocker.Mock() + mock_libvirt = mocker.Mock() + mock_libvirt.libvirtError = FakeError + mock_libvirt.VIR_ERR_NO_SECRET = 42 + secret_mock = mocker.Mock() + secret_mock.value.side_effect = FakeError() + mock_get_libvirt = mocker.patch.object(hooks, "_get_libvirt") + mock_get_libvirt.return_value = mock_libvirt + mock_libvirt.open.return_value = conn_mock + mock_set_secret = mocker.patch.object(hooks, "_set_secret") + conn_mock.listSecrets.return_value = ["uuid1"] + conn_mock.secretLookupByUUIDString.return_value = secret_mock + hooks._ensure_secret("uuid1", "secret") + mock_set_secret.assert_called_once_with(conn_mock, "uuid1", "secret")