From f130494def06390c75520b3a73f1261c9b9f3101 Mon Sep 17 00:00:00 2001 From: AlexCK-STFC <210735915+AlexCK-STFC@users.noreply.github.com> Date: Wed, 1 Oct 2025 11:46:47 +0100 Subject: [PATCH 1/3] ENH: Add support for PCPUs It's just been combined into the existing VCPUs field + more aliases adding --- docs/user_docs/query_docs/HYPERVISORS.md | 36 +++++++++---------- .../enums/props/hypervisor_properties.py | 29 +++++++++++---- openstackquery/runners/hypervisor_runner.py | 10 ++++-- .../structs/resource_provider_usage.py | 4 +++ .../enums/props/test_hypervisor_properties.py | 28 ++++++++++++--- tests/runners/test_hypervisor_runner.py | 27 +++++++++++--- 6 files changed, 97 insertions(+), 37 deletions(-) diff --git a/docs/user_docs/query_docs/HYPERVISORS.md b/docs/user_docs/query_docs/HYPERVISORS.md index cac3b33..ac9a983 100644 --- a/docs/user_docs/query_docs/HYPERVISORS.md +++ b/docs/user_docs/query_docs/HYPERVISORS.md @@ -19,24 +19,24 @@ from openstackquery import HypervisorQuery `Hypervisors` have the following properties: -| Return Type | Property Name(s) (case-insensitive) | Description | -|-------------|------------------------------------------------------------------|-----------------------------------------------------------------| -| `int` | "disk_gb_avail", "disk_avail", "local_disk_free", "free_disk_gb" | The local disk space remaining on this hypervisor(in GiB) | -| `int` | "disk_gb_used", "disk_used", "local_disk_used", "local_gb_used" | The local disk space allocated on this hypervisor(in GiB) | -| `int` | "disk_gb_size", "disk", "local_disk", "local_gb" | The total amount of local disk space(in GiB) | -| `string` | "id", "uuid", "host_id" | ID of the Hypervisor | -| `string` | "ip", "host_ip" | The IP address of the hypervisor’s host | -| `int` | "memory_mb_avail", "memory_avail", "memory_free", "free_ram_mb" | The free RAM on this hypervisor(in MiB). | -| `int` | "memory_used", "memory_mb_used" | RAM currently being used on this hypervisor(in MiB). | -| `int` | "memory_mb_size", "memory_size", "memory_mb", "memory", "ram" | The total amount of ram(in MiB) | -| `string` | "name", "host_name" | Hypervisor Hostname | -| `string` | "state" | The state of the hypervisor. One of up or down. | -| `string` | "status" | The status of the hypervisor. One of enabled or disabled. | -| `int` | "vcpus" | The number of vCPUs on this hypervisor. | -| `int` | "vcpus_free" "vcpus_avail" | The number of vCPUs on this hypervisor. | -| `int` | "vcpus_used", "vcpus_in_use" | The number of vCPUs currently being used on this hypervisor. | -| `string` | "disabled_reason" | Comment of why the hypervisor is disabled, None if not disabled | -| `float` | "uptime" | The total uptime in days of the hypervisor | +| Return Type | Property Name(s) (case-insensitive) | Description | +|-------------|----------------------------------------------------------------------------------------|--------------------------------------------------------------------| +| `int` | "disk_gb_avail", "disk_avail", "local_disk_free", "free_disk_gb" | The local disk space remaining on this hypervisor(in GiB) | +| `int` | "disk_gb_used", "disk_used", "local_disk_used", "local_gb_used" | The local disk space allocated on this hypervisor(in GiB) | +| `int` | "disk_gb_size", "disk", "local_disk", "local_gb" | The total amount of local disk space(in GiB) | +| `string` | "id", "uuid", "host_id" | ID of the Hypervisor | +| `string` | "ip", "host_ip" | The IP address of the hypervisor’s host | +| `int` | "memory_mb_avail", "memory_avail", "memory_free", "free_ram_mb" | The free RAM on this hypervisor(in MiB). | +| `int` | "memory_used", "memory_mb_used" | RAM currently being used on this hypervisor(in MiB). | +| `int` | "memory_mb_size", "memory_size", "memory_mb", "memory", "ram" | The total amount of ram(in MiB) | +| `string` | "name", "host_name" | Hypervisor Hostname | +| `string` | "state" | The state of the hypervisor. One of up or down. | +| `string` | "status" | The status of the hypervisor. One of enabled or disabled. | +| `int` | "vcpus", "cpus", "pcpus" | The number of vCPUs/pCPUs on this hypervisor. | +| `int` | "vcpus_free", "vcpus_avail", "cpus_avail", "cpus_free", "pcpus_avail", "pcpus_free" | The number of vCPUs/pCPUs on this hypervisor. | +| `int` | "vcpus_used", "vcpus_in_use", "cpus_used", "cpus_in_use", "pcpus_used", "pcpus_in_use" | The number of vCPUs/pCPUs currently being used on this hypervisor. | +| `string` | "disabled_reason" | Comment of why the hypervisor is disabled, None if not disabled | +| `float` | "uptime" | The total uptime in days of the hypervisor | Any of these properties can be used for any of the API methods that takes a property - like `select`, `where`, `sort_by` etc diff --git a/openstackquery/enums/props/hypervisor_properties.py b/openstackquery/enums/props/hypervisor_properties.py index 371b146..ce53dcb 100644 --- a/openstackquery/enums/props/hypervisor_properties.py +++ b/openstackquery/enums/props/hypervisor_properties.py @@ -43,9 +43,23 @@ def _get_aliases() -> Dict: HypervisorProperties.HYPERVISOR_STATUS: ["status"], HypervisorProperties.HYPERVISOR_DISABLED_REASON: ["disabled_reason"], HypervisorProperties.HYPERVISOR_UPTIME_DAYS: ["uptime"], - HypervisorProperties.VCPUS: ["vcpus"], - HypervisorProperties.VCPUS_USED: ["vcpus_used", "vcpus_in_use"], - HypervisorProperties.VCPUS_AVAIL: ["vcpus_avail", "vcpus_free"], + HypervisorProperties.VCPUS: ["vcpus", "cpus", "pcpus"], + HypervisorProperties.VCPUS_USED: [ + "vcpus_used", + "vcpus_in_use", + "cpus_used", + "cpus_in_use", + "pcpus_used", + "pcpus_in_use", + ], + HypervisorProperties.VCPUS_AVAIL: [ + "vcpus_avail", + "vcpus_free", + "cpus_avail", + "cpus_free", + "pcpus_avail", + "pcpus_free", + ], HypervisorProperties.MEMORY_MB_SIZE: [ "memory_mb_size", "memory_size", @@ -92,7 +106,6 @@ def get_prop_mapping(prop) -> Optional[PropFunc]: HypervisorProperties.HYPERVISOR_ID: lambda a: a.hv["id"], HypervisorProperties.HYPERVISOR_IP: lambda a: a.hv["host_ip"], HypervisorProperties.HYPERVISOR_NAME: lambda a: a.hv["name"], - # HypervisorProperties.HYPERVISOR_SERVER_COUNT: lambda a: a["runnning_vms"], HypervisorProperties.HYPERVISOR_STATE: lambda a: a.hv["state"], HypervisorProperties.HYPERVISOR_STATUS: lambda a: a.hv["status"], HypervisorProperties.HYPERVISOR_DISABLED_REASON: lambda a: a.hv["service"][ @@ -101,13 +114,15 @@ def get_prop_mapping(prop) -> Optional[PropFunc]: HypervisorProperties.HYPERVISOR_UPTIME_DAYS: lambda a: TimeUtils.extract_uptime( a.hv["uptime"] ), - HypervisorProperties.VCPUS: lambda a: a.usage.vcpus, - HypervisorProperties.VCPUS_AVAIL: lambda a: a.usage.vcpus_avail, + HypervisorProperties.VCPUS: lambda a: a.usage.vcpus + a.usage.pcpus, + HypervisorProperties.VCPUS_AVAIL: lambda a: a.usage.vcpus_avail + + a.usage.pcpus_avail, + HypervisorProperties.VCPUS_USED: lambda a: a.usage.vcpus_used + + a.usage.pcpus_used, HypervisorProperties.MEMORY_MB_SIZE: lambda a: a.usage.memory_mb_size, HypervisorProperties.MEMORY_MB_AVAIL: lambda a: a.usage.memory_mb_avail, HypervisorProperties.DISK_GB_SIZE: lambda a: a.usage.disk_gb_size, HypervisorProperties.DISK_GB_AVAIL: lambda a: a.usage.disk_gb_avail, - HypervisorProperties.VCPUS_USED: lambda a: a.usage.vcpus_used, HypervisorProperties.MEMORY_MB_USED: lambda a: a.usage.memory_mb_used, HypervisorProperties.DISK_GB_USED: lambda a: a.usage.disk_gb_used, } diff --git a/openstackquery/runners/hypervisor_runner.py b/openstackquery/runners/hypervisor_runner.py index 1ef2032..bcccc7d 100644 --- a/openstackquery/runners/hypervisor_runner.py +++ b/openstackquery/runners/hypervisor_runner.py @@ -1,5 +1,5 @@ import logging -from typing import List, Optional, Dict +from typing import Dict, List, Optional from openstack import exceptions, utils from openstack.compute.v2.hypervisor import Hypervisor as OpenstackHypervisor @@ -9,7 +9,6 @@ from openstackquery.openstack_connection import OpenstackConnection from openstackquery.runners.runner_utils import RunnerUtils from openstackquery.runners.runner_wrapper import RunnerWrapper - from openstackquery.structs.hypervisor import Hypervisor from openstackquery.structs.resource_provider_usage import ResourceProviderUsage @@ -69,6 +68,8 @@ def _convert_to_custom_obj( memory_mb_used = usage.get("MEMORY_MB", 0) disk_gb_used = usage.get("DISK_GB", 0) + pcpus_used = usage.get("PCPU", 0) + return ResourceProviderUsage( # workaround for hvs not containing VCPU/Memory/Disk resource provider info - set to 0 vcpus_used=vcpus_used, @@ -80,6 +81,9 @@ def _convert_to_custom_obj( vcpus=avail["VCPU"] + vcpus_used, memory_mb_size=avail["MEMORY_MB"] + memory_mb_used, disk_gb_size=avail["DISK_GB"] + disk_gb_used, + pcpus_avail=avail["PCPU"], + pcpus_used=pcpus_used, + pcpus=pcpus_used + avail["PCPU"], ) @staticmethod @@ -94,7 +98,7 @@ def _get_availability_info( :return: A dictionary with the summed availability stats using the class name as a key """ summed_classes = {} - for resource_class in ["VCPU", "MEMORY_MB", "DISK_GB"]: + for resource_class in ["VCPU", "MEMORY_MB", "DISK_GB", "PCPU"]: placement_inventories = conn.placement.resource_provider_inventories( resource_provider_obj, resource_class=resource_class ) diff --git a/openstackquery/structs/resource_provider_usage.py b/openstackquery/structs/resource_provider_usage.py index e9071a5..65d5534 100644 --- a/openstackquery/structs/resource_provider_usage.py +++ b/openstackquery/structs/resource_provider_usage.py @@ -21,3 +21,7 @@ class ResourceProviderUsage: vcpus_used: int memory_mb_used: int disk_gb_used: int + + pcpus: int + pcpus_used: int + pcpus_avail: int diff --git a/tests/enums/props/test_hypervisor_properties.py b/tests/enums/props/test_hypervisor_properties.py index 09f85b8..578caa9 100644 --- a/tests/enums/props/test_hypervisor_properties.py +++ b/tests/enums/props/test_hypervisor_properties.py @@ -1,11 +1,11 @@ from unittest.mock import patch + import pytest from openstackquery.enums.props.hypervisor_properties import HypervisorProperties from openstackquery.exceptions.query_property_mapping_error import ( QueryPropertyMappingError, ) - from tests.mocks.mocked_props import MockProperties @@ -28,7 +28,17 @@ ["hypervisor_disabled_reason", "disabled_reason"], ), (HypervisorProperties.HYPERVISOR_UPTIME_DAYS, ["hypervisor_uptime_days"]), - (HypervisorProperties.VCPUS_AVAIL, ["vcpus_avail"]), + ( + HypervisorProperties.VCPUS_AVAIL, + [ + "vcpus_avail", + "vcpus_free", + "cpus_avail", + "cpus_free", + "pcpus_avail", + "pcpus_free", + ], + ), ( HypervisorProperties.MEMORY_MB_AVAIL, ["memory_mb_avail", "memory_avail", "memory_free", "free_ram_mb"], @@ -37,7 +47,17 @@ HypervisorProperties.DISK_GB_AVAIL, ["disk_gb_avail", "disk_avail", "local_disk_free", "free_disk_gb"], ), - (HypervisorProperties.VCPUS_USED, ["vcpus_used", "vcpus_in_use"]), + ( + HypervisorProperties.VCPUS_USED, + [ + "vcpus_used", + "vcpus_in_use", + "cpus_used", + "cpus_in_use", + "pcpus_used", + "pcpus_in_use", + ], + ), (HypervisorProperties.MEMORY_MB_USED, ["memory_mb_used", "memory_used"]), ( HypervisorProperties.DISK_GB_USED, @@ -51,7 +71,7 @@ HypervisorProperties.MEMORY_MB_SIZE, ["memory_mb_size", "memory_size", "memory_mb", "memory", "ram"], ), - (HypervisorProperties.VCPUS, ["vcpus"]), + (HypervisorProperties.VCPUS, ["vcpus", "cpus", "pcpus"]), ], ) def test_property_serialization(expected_prop, test_values, property_variant_generator): diff --git a/tests/runners/test_hypervisor_runner.py b/tests/runners/test_hypervisor_runner.py index 283406e..bc26c7f 100644 --- a/tests/runners/test_hypervisor_runner.py +++ b/tests/runners/test_hypervisor_runner.py @@ -1,8 +1,9 @@ -from unittest.mock import MagicMock, NonCallableMock, patch, call +from unittest.mock import MagicMock, NonCallableMock, call, patch + import pytest -from openstackquery.structs.resource_provider_usage import ResourceProviderUsage from openstackquery.runners.hypervisor_runner import HypervisorRunner +from openstackquery.structs.resource_provider_usage import ResourceProviderUsage @pytest.fixture(name="instance") @@ -89,11 +90,12 @@ def _mock_resource_provider_inventories(_, resource_class): # test case for single provider with all inventory resources" { "providers": [{"id": "id1", "name": "foo"}], - "usages": {"VCPU": 4, "MEMORY_MB": 8192, "DISK_GB": 100}, + "usages": {"VCPU": 4, "MEMORY_MB": 8192, "DISK_GB": 100, "PCPU": 48}, "inventories": { "VCPU": [{"total": 16}], "MEMORY_MB": [{"total": 32768}], "DISK_GB": [{"total": 500}], + "PCPU": [{"total": 16}], }, "expected_results": { "foo": ResourceProviderUsage( @@ -106,13 +108,21 @@ def _mock_resource_provider_inventories(_, resource_class): vcpus=20, memory_mb_size=40960, disk_gb_size=600, + pcpus_used=48, + pcpus_avail=16, + pcpus=64, ) }, }, # "test case for single provider with no inventory resources", { "providers": [{"id": "id1", "name": "foo"}], - "usages": {"VCPU": 0, "MEMORY_MB": 0, "DISK_GB": 0}, + "usages": { + "VCPU": 0, + "MEMORY_MB": 0, + "DISK_GB": 0, + "PCPU": 0, + }, "inventories": {}, "expected_results": { "foo": ResourceProviderUsage( @@ -125,6 +135,9 @@ def _mock_resource_provider_inventories(_, resource_class): vcpus=0, memory_mb_size=0, disk_gb_size=0, + pcpus_used=0, + pcpus_avail=0, + pcpus=0, ) }, }, @@ -136,11 +149,12 @@ def _mock_resource_provider_inventories(_, resource_class): {"id": "id-1", "name": "provider-1"}, {"id": "id-2", "name": "provider-2"}, ], - "usages": {"VCPU": 4, "MEMORY_MB": 8192, "DISK_GB": 100}, + "usages": {"VCPU": 4, "MEMORY_MB": 8192, "DISK_GB": 100, "PCPU": 32}, "inventories": { "VCPU": [{"total": 16}, {"total": 16}], "MEMORY_MB": [{"total": 500}, {"total": 1000}], "DISK_GB": [{"total": 500}, {"total": 1500}], + "PCPU": [{"total": 4}, {"total": 12}], }, "expected_results": { f"provider-{i}": ResourceProviderUsage( @@ -153,6 +167,9 @@ def _mock_resource_provider_inventories(_, resource_class): vcpus=36, memory_mb_size=9692, disk_gb_size=2100, + pcpus_avail=16, + pcpus_used=32, + pcpus=48, ) for i in range(3) }, From c0920b81d5aa98be0c307950ecc414f2c01950b2 Mon Sep 17 00:00:00 2001 From: AlexCK-STFC <210735915+AlexCK-STFC@users.noreply.github.com> Date: Wed, 1 Oct 2025 11:51:12 +0100 Subject: [PATCH 2/3] MAINT: Bump minor Addition of pcpus to vcpus is backwards-compatibile --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2099f17..7738dbe 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name="openstackquery", - version="2.0.0", + version="2.1.0", author="STFC Cloud Team", author_email="", description=DESCRIPTION, From fae8faec3655bef95865fc688c17d06f2d1c8f9d Mon Sep 17 00:00:00 2001 From: AlexCK-STFC <210735915+AlexCK-STFC@users.noreply.github.com> Date: Wed, 1 Oct 2025 13:32:46 +0100 Subject: [PATCH 3/3] TST: Add test coverage for logic in hypervisor prop mapping --- .../enums/props/test_hypervisor_properties.py | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/tests/enums/props/test_hypervisor_properties.py b/tests/enums/props/test_hypervisor_properties.py index 578caa9..d6f8194 100644 --- a/tests/enums/props/test_hypervisor_properties.py +++ b/tests/enums/props/test_hypervisor_properties.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass from unittest.mock import patch import pytest @@ -106,3 +107,122 @@ def test_get_marker_prop_func(mock_get_prop_mapping): val = HypervisorProperties.get_marker_prop_func() mock_get_prop_mapping.assert_called_once_with(HypervisorProperties.HYPERVISOR_ID) assert val == mock_get_prop_mapping.return_value + + +# pylint: disable=too-many-instance-attributes +@dataclass +class MockUsage: + vcpus: int = 0 + pcpus: int = 0 + vcpus_used: int = 0 + pcpus_used: int = 0 + vcpus_avail: int = 0 + pcpus_avail: int = 0 + memory_mb_size: int = 0 + memory_mb_used: int = 0 + memory_mb_avail: int = 0 + disk_gb_size: int = 0 + disk_gb_used: int = 0 + disk_gb_avail: int = 0 + + +@dataclass +class MockHypervisor: + hv: dict + usage: MockUsage + + +@pytest.mark.parametrize( + "prop, mock_hv, mock_usage, expected", + [ + (HypervisorProperties.HYPERVISOR_ID, {"id": "abc123"}, None, "abc123"), + (HypervisorProperties.HYPERVISOR_IP, {"host_ip": "10.0.0.1"}, None, "10.0.0.1"), + (HypervisorProperties.HYPERVISOR_NAME, {"name": "hyp1"}, None, "hyp1"), + (HypervisorProperties.HYPERVISOR_STATE, {"state": "up"}, None, "up"), + ( + HypervisorProperties.HYPERVISOR_STATUS, + {"status": "enabled"}, + None, + "enabled", + ), + ( + HypervisorProperties.HYPERVISOR_DISABLED_REASON, + {"service": {"disabled_reason": "maintenance"}}, + None, + "maintenance", + ), + ( + HypervisorProperties.VCPUS, + {}, + MockUsage(vcpus=4, pcpus=2), + 6, + ), + ( + HypervisorProperties.VCPUS_USED, + {}, + MockUsage(vcpus_used=1, pcpus_used=2), + 3, + ), + ( + HypervisorProperties.VCPUS_AVAIL, + {}, + MockUsage(vcpus_avail=3, pcpus_avail=1), + 4, + ), + ( + HypervisorProperties.MEMORY_MB_SIZE, + {}, + MockUsage(memory_mb_size=8192), + 8192, + ), + ( + HypervisorProperties.MEMORY_MB_USED, + {}, + MockUsage(memory_mb_used=4096), + 4096, + ), + ( + HypervisorProperties.MEMORY_MB_AVAIL, + {}, + MockUsage(memory_mb_avail=2048), + 2048, + ), + ( + HypervisorProperties.DISK_GB_SIZE, + {}, + MockUsage(disk_gb_size=500), + 500, + ), + ( + HypervisorProperties.DISK_GB_USED, + {}, + MockUsage(disk_gb_used=200), + 200, + ), + ( + HypervisorProperties.DISK_GB_AVAIL, + {}, + MockUsage(disk_gb_avail=300), + 300, + ), + ], +) +def test_hypervisor_property_mappings(prop, mock_hv, mock_usage, expected): + """Test that each HypervisorProperties mapping correctly extracts the expected data.""" + hv_obj = MockHypervisor(mock_hv, mock_usage) + func = HypervisorProperties.get_prop_mapping(prop) + assert func(hv_obj) == expected + + +@patch("openstackquery.enums.props.hypervisor_properties.TimeUtils.extract_uptime") +def test_hypervisor_uptime_days_mapping(mock_extract): + mock_extract.return_value = 5 + hv_obj = MockHypervisor({"uptime": "fake-uptime-string"}, None) + + func = HypervisorProperties.get_prop_mapping( + HypervisorProperties.HYPERVISOR_UPTIME_DAYS + ) + result = func(hv_obj) + + mock_extract.assert_called_once_with("fake-uptime-string") + assert result == 5