From 3114f6e7b5b25a4152db84391201524a7e8197f9 Mon Sep 17 00:00:00 2001 From: josepselga Date: Mon, 24 Nov 2025 22:39:41 +0100 Subject: [PATCH 1/2] F OpenNebula/one#7385: OpenNebula FabricManager - Automatically recover last configured partitions (Stateful appliance) Signed-off-by: josepselga --- appliances/FabricManager/appliance.sh | 61 +++++++++---- .../partitioner.cpp | 86 +++++++++++++++++-- .../FabricManager/one-fm-boot-manager.sh | 84 ++++++++++++++++++ .../FabricManager.pkr.hcl | 5 ++ 4 files changed, 214 insertions(+), 22 deletions(-) create mode 100644 appliances/FabricManager/one-fm-boot-manager.sh diff --git a/appliances/FabricManager/appliance.sh b/appliances/FabricManager/appliance.sh index 135dea10..d3e0c88f 100644 --- a/appliances/FabricManager/appliance.sh +++ b/appliances/FabricManager/appliance.sh @@ -19,10 +19,7 @@ set -o errexit -o pipefail ### Important notes ################################################## # # 1. This appliance requires a base OS image with cloud-init and wget. -# 2. You MUST edit the 'PARTITION_TOOL_SRC' variable in the 'Globals' -# section below to point to the 'partitions.cpp' file in your -# internal GitLab repository. -# 3. This appliance MUST be instantiated with a VM Template that +# 2. This appliance MUST be instantiated with a VM Template that # includes UEFI boot and Q35 machine settings . # ### Important notes ################################################## @@ -44,7 +41,7 @@ ONE_SERVICE_SHORT_DESCRIPTION='Appliance with NVIDIA Fabric Manager for Shared N ONE_SERVICE_DESCRIPTION=$(cat < "${OVERRIDE_DIR}/override.conf" +[Service] +ExecStart= +ExecStart=/usr/local/sbin/one-fm-boot-manager.sh +EOF + + systemctl daemon-reload msg info "Enabling nvidia-fabricmanager systemd service" systemctl enable nvidia-fabricmanager.service diff --git a/appliances/FabricManager/fabricManager-partition-tool/partitioner.cpp b/appliances/FabricManager/fabricManager-partition-tool/partitioner.cpp index 51f4c59b..623a052a 100644 --- a/appliances/FabricManager/fabricManager-partition-tool/partitioner.cpp +++ b/appliances/FabricManager/fabricManager-partition-tool/partitioner.cpp @@ -22,9 +22,50 @@ #include #include #include +#include +#include +#include #include "nv_fm_agent.h" +const char* STATE_FILE_PATH = "/var/lib/nvidia-fabricmanager/active_partitions.state"; + +// --- State Management Functions --- + +std::vector readState() { + std::vector activePartitions; + std::ifstream stateFile(STATE_FILE_PATH); + fmFabricPartitionId_t id; + while (stateFile >> id) { + activePartitions.push_back(id); + } + return activePartitions; +} + +void writeState(const std::vector& activePartitions) { + std::ofstream stateFile(STATE_FILE_PATH, std::ios::trunc); + for (const auto& id : activePartitions) { + stateFile << id << std::endl; + } +} + +void updateStateFile(fmFabricPartitionId_t partitionId, bool activate) { + auto activePartitions = readState(); + auto it = std::find(activePartitions.begin(), activePartitions.end(), partitionId); + + if (activate) { + if (it == activePartitions.end()) { + activePartitions.push_back(partitionId); + } + } else { + if (it != activePartitions.end()) { + activePartitions.erase(it); + } + } + writeState(activePartitions); +} + + void printFmError(const char* operation, fmReturn_t fmReturn) { std::cout << "Error: Failed to " << operation << ". (Code: " << fmReturn << ")" << std::endl; } @@ -40,7 +81,8 @@ void printMenu() { std::cout << " 0 - List Supported Partitions\n"; std::cout << " 1 - Activate a Partition\n"; std::cout << " 2 - Deactivate a Partition\n"; - std::cout << " 3 - Quit\n"; + std::cout << " 3 - Restore Active Partitions\n"; + std::cout << " 4 - Quit\n"; std::cout << "------------------------------------------\n"; std::cout << "Enter operation: "; } @@ -142,6 +184,7 @@ fmReturn_t executeOperation(fmHandle_t fmHandle, unsigned int operation, fmFabri fmReturn = fmActivateFabricPartition(fmHandle, partitionId); if (fmReturn == FM_ST_SUCCESS) { std::cout << "Successfully sent activation request for partition " << partitionId << std::endl; + updateStateFile(partitionId, true); } else { printFmError("activate partition", fmReturn); } @@ -156,12 +199,41 @@ fmReturn_t executeOperation(fmHandle_t fmHandle, unsigned int operation, fmFabri fmReturn = fmDeactivateFabricPartition(fmHandle, partitionId); if (fmReturn == FM_ST_SUCCESS) { std::cout << "Successfully sent deactivation request for partition " << partitionId << std::endl; + updateStateFile(partitionId, false); } else { printFmError("deactivate partition", fmReturn); } break; } + case 3: { // Restore Active Partitions + auto activePartitions = readState(); + + fmActivatedFabricPartitionList_t partitionsToRestore = {0}; + partitionsToRestore.version = fmActivatedFabricPartitionList_version; + + if (activePartitions.size() > FM_MAX_FABRIC_PARTITIONS) { + std::cout << "Error: Number of active partitions in state file exceeds limit." << std::endl; + return FM_ST_BADPARAM; + } + + partitionsToRestore.numPartitions = activePartitions.size(); + for(size_t i = 0; i < activePartitions.size(); ++i) { + partitionsToRestore.partitionIds[i] = activePartitions[i]; + } + + std::cout << "Restoring " << partitionsToRestore.numPartitions << " active partition(s)..." << std::endl; + + fmReturn = fmSetActivatedFabricPartitions(fmHandle, &partitionsToRestore); + + if (fmReturn == FM_ST_SUCCESS) { + std::cout << "Successfully restored active partitions." << std::endl; + } else { + printFmError("restore active partitions", fmReturn); + } + break; + } + default: std::cout << "Error: Invalid operation specified (" << operation << ")." << std::endl; fmReturn = FM_ST_BADPARAM; @@ -214,10 +286,10 @@ int main(int argc, char **argv) runInteractive = false; } else if (!runInteractive) { std::cout << "Usage: " << argv[0] << " [-i ] -o [-p ] [-f ]\n" - << " -i, --ip : IP address of Fabric Manager (default: 127.0.0.1)\n" - << " -o, --operation : 0=List, 1=Activate, 2=Deactivate\n" - << " -p, --partition : Partition ID (required for Activate/Deactivate)\n" - << " -f, --format : Output format for operation 0 (csv or table, default: table)\n" + << " -i, --ip : IP address of Fabric Manager (default: 127.0.0.1)\n" + << " -o, --operation : 0=List, 1=Activate, 2=Deactivate, 3=Restore\n" + << " -p, --partition : Partition ID (for Activate/Deactivate)\n" + << " -f, --format : Output format for op 0 (csv or table, default: table)\n" << "Running without options starts interactive mode.\n"; return FM_ST_BADPARAM; } @@ -259,7 +331,7 @@ int main(int argc, char **argv) return fmReturn; } - if (outputFormat != "csv") { + if (outputFormat != "csv" && operation != 3) { std::cout << "Successfully connected to Fabric Manager at " << hostIpAddress << std::endl; } @@ -277,7 +349,7 @@ int main(int argc, char **argv) continue; } - if (operation == 3) break; + if (operation == 4) break; partitionId = PARTITION_ID_NOT_SET; if (operation == 1 || operation == 2) { diff --git a/appliances/FabricManager/one-fm-boot-manager.sh b/appliances/FabricManager/one-fm-boot-manager.sh new file mode 100644 index 00000000..d01d9c11 --- /dev/null +++ b/appliances/FabricManager/one-fm-boot-manager.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# ---------------------------------------------------------------------------- # +# Copyright 2024, OpenNebula Project, OpenNebula Systems # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +# ---------------------------------------------------------------------------- # + +# +# Script that manages the startup and state restoration of nvidia-fabricmanager +# + +set -o errexit -o pipefail + +FM_STATE_FILE="/var/lib/nvidia-fabricmanager/fabricmanager.state" +ONE_STATE_FILE="/var/lib/nvidia-fabricmanager/active_partitions.state" +PARTITIONER_TOOL="/usr/local/sbin/nv-partitioner" +LOG_TAG="one-fm-boot-manager" + +log() { + echo "$@" >&2 + logger -t "${LOG_TAG}" -- "$@" +} + +log "Starting NVIDIA Fabric Manager boot manager..." + +# Handle the case where one state file exists, but NVIDIA's does not. +# This is an inconsistent state. We back up our file and start fresh. +if [ -s "${ONE_STATE_FILE}" ] && [ ! -f "${FM_STATE_FILE}" ]; then + BACKUP_FILE="${ONE_STATE_FILE}.failed-$(date +%Y%m%d-%H%M%S)" + log "WARNING: Inconsistent state detected. NVIDIA's state file is missing, but OpenNebula's partition state file exists." + log "WARNING: Backing up current partition state to ${BACKUP_FILE}" + mv "${ONE_STATE_FILE}" "${BACKUP_FILE}" + log "WARNING: Proceeding with a fresh start. No partitions will be restored." +fi + +# 1. Decide which mode to start the Fabric Manager +if [ -f "${FM_STATE_FILE}" ]; then + log "State file found. Starting Fabric Manager in --restart mode." + /usr/bin/nv-fabricmanager --restart & +else + log "No state file found. Starting Fabric Manager in normal mode." + /usr/bin/nv-fabricmanager & +fi + +# Capture the PID of the last background process +FM_PID=$! +log "Fabric Manager daemon started with PID ${FM_PID}." + +# 2. Wait for the daemon to be ready +log "Waiting for 5 seconds for the daemon to initialize..." +sleep 5 + +# 3. Perform the partition state restoration. +# ONLY attempt restore if we started in restart mode (i.e., FM_STATE_FILE existed) +if [ -f "${FM_STATE_FILE}" ]; then + if [ -x "${PARTITIONER_TOOL}" ]; then + log "Executing partition restore operation..." + if ! "${PARTITIONER_TOOL}" -o 3; then + log "WARNING: Partition restore command failed. Check fabricmanager logs." + else + log "Partition restore operation completed." + fi + else + log "WARNING: Partitioner tool not found at ${PARTITIONER_TOOL}. Skipping restore." + fi +else + log "INFO: Started in normal mode (FM state file missing), skipping partition restore operation." +fi + +# 4. Wait for the Fabric Manager daemon to exit. +log "Boot manager script is now waiting for the daemon to exit." +wait "${FM_PID}" + +exit $? diff --git a/packer/service_FabricManager/FabricManager.pkr.hcl b/packer/service_FabricManager/FabricManager.pkr.hcl index 4196be8d..fef71dd6 100644 --- a/packer/service_FabricManager/FabricManager.pkr.hcl +++ b/packer/service_FabricManager/FabricManager.pkr.hcl @@ -93,6 +93,11 @@ build { destination = "/etc/one-appliance/fabricManager-partition-tool" } + provisioner "file" { + source = "appliances/FabricManager/one-fm-boot-manager.sh" + destination = "/etc/one-appliance/one-fm-boot-manager.sh" + } + provisioner "shell" { scripts = ["${var.input_dir}/82-configure-context.sh"] } From 5aeea4176d244629224084ee19dce4fbf172a10a Mon Sep 17 00:00:00 2001 From: josepselga Date: Thu, 27 Nov 2025 11:02:34 +0100 Subject: [PATCH 2/2] F OpenNebula/one#7385: OpenNebula FabricManager - Automatically recover last configured partitions (Stateful appliance) - Hard recovery Mode Signed-off-by: josepselga --- .../FabricManager/one-fm-boot-manager.sh | 113 +++++++++++++----- 1 file changed, 82 insertions(+), 31 deletions(-) diff --git a/appliances/FabricManager/one-fm-boot-manager.sh b/appliances/FabricManager/one-fm-boot-manager.sh index d01d9c11..52861c23 100644 --- a/appliances/FabricManager/one-fm-boot-manager.sh +++ b/appliances/FabricManager/one-fm-boot-manager.sh @@ -33,48 +33,99 @@ log() { log "Starting NVIDIA Fabric Manager boot manager..." -# Handle the case where one state file exists, but NVIDIA's does not. -# This is an inconsistent state. We back up our file and start fresh. +# Check for inconsistent state before starting Fabric Manager if [ -s "${ONE_STATE_FILE}" ] && [ ! -f "${FM_STATE_FILE}" ]; then + # --- Hard Recovery --- + log "WARNING: Inconsistent state detected. Starting hard recovery process." + + # 1. Create backup BACKUP_FILE="${ONE_STATE_FILE}.failed-$(date +%Y%m%d-%H%M%S)" - log "WARNING: Inconsistent state detected. NVIDIA's state file is missing, but OpenNebula's partition state file exists." - log "WARNING: Backing up current partition state to ${BACKUP_FILE}" + log "INFO: Backing up current partition state to ${BACKUP_FILE}" mv "${ONE_STATE_FILE}" "${BACKUP_FILE}" - log "WARNING: Proceeding with a fresh start. No partitions will be restored." -fi -# 1. Decide which mode to start the Fabric Manager -if [ -f "${FM_STATE_FILE}" ]; then - log "State file found. Starting Fabric Manager in --restart mode." - /usr/bin/nv-fabricmanager --restart & -else - log "No state file found. Starting Fabric Manager in normal mode." + # 2. Start FM in normal mode + log "INFO: Starting Fabric Manager in normal mode for recovery." /usr/bin/nv-fabricmanager & -fi + FM_PID=$! + log "INFO: Fabric Manager daemon started with PID ${FM_PID}." + log "INFO: Waiting for 5 seconds for the daemon to initialize..." + sleep 5 + + # 3. Read partitions from backup and activate one-by-one + log "INFO: Attempting to reactivate partitions from ${BACKUP_FILE}" + while read -r PARTITION_ID; do + # Skip empty + if [ -z "$PARTITION_ID" ]; then continue; fi -# Capture the PID of the last background process -FM_PID=$! -log "Fabric Manager daemon started with PID ${FM_PID}." - -# 2. Wait for the daemon to be ready -log "Waiting for 5 seconds for the daemon to initialize..." -sleep 5 - -# 3. Perform the partition state restoration. -# ONLY attempt restore if we started in restart mode (i.e., FM_STATE_FILE existed) -if [ -f "${FM_STATE_FILE}" ]; then - if [ -x "${PARTITIONER_TOOL}" ]; then - log "Executing partition restore operation..." - if ! "${PARTITIONER_TOOL}" -o 3; then - log "WARNING: Partition restore command failed. Check fabricmanager logs." + log "INFO: Attempting to activate partition ID: ${PARTITION_ID}" + if "${PARTITIONER_TOOL}" -o 1 -p "${PARTITION_ID}"; then + log "SUCCESS: Partition ${PARTITION_ID} activated." else - log "Partition restore operation completed." + log "ERROR: Failed to activate partition ${PARTITION_ID}. Check logs for details." fi + done < "${BACKUP_FILE}" + + # 4. Final Summary + log "INFO: Hard recovery process finished. Validating final state..." + + # Ensure the new state file exists + if [ ! -f "${ONE_STATE_FILE}" ]; then + touch "${ONE_STATE_FILE}" + fi + + # compare files + SORTED_BACKUP=$(mktemp) + SORTED_CURRENT=$(mktemp) + sort "${BACKUP_FILE}" > "${SORTED_BACKUP}" + sort "${ONE_STATE_FILE}" > "${SORTED_CURRENT}" + if diff -q "${SORTED_BACKUP}" "${SORTED_CURRENT}" >/dev/null; then + log "SUCCESS: Hard recovery complete. All partitions were successfully restored." else - log "WARNING: Partitioner tool not found at ${PARTITIONER_TOOL}. Skipping restore." + log "CRITICAL: Hard recovery was PARTIAL. Not all partitions could be restored." + log "CRITICAL: The following differences were found between the desired state (left) and the recovered state (right):" + diff "${SORTED_BACKUP}" "${SORTED_CURRENT}" | logger -t "${LOG_TAG}" -- fi + + rm -f "${SORTED_BACKUP}" "${SORTED_CURRENT}" + else - log "INFO: Started in normal mode (FM state file missing), skipping partition restore operation." + # --- Normal/Resilient Boot --- + log "INFO: Consistent state detected. Proceeding with normal or restart boot." + + RESTART_MODE=false + + # 1. Decide which mode to start the Fabric Manager based on the saved decision + if [ -f "${FM_STATE_FILE}" ] && [ -s "${ONE_STATE_FILE}" ]; then + log "State file found. Starting Fabric Manager in --restart mode." + RESTART_MODE=true + /usr/bin/nv-fabricmanager --restart & + else + log "No state file found. Starting Fabric Manager in normal mode." + /usr/bin/nv-fabricmanager & + fi + + FM_PID=$! + log "Fabric Manager daemon started with PID ${FM_PID}." + + # 2. Wait for the daemon to be ready + log "Waiting for 5 seconds for the daemon to initialize..." + sleep 5 + + # 3. Perform partition restoration based on the SAVED start-up decision + if [ "${RESTART_MODE}" = true ]; then + if [ -x "${PARTITIONER_TOOL}" ]; then + log "Executing atomic partition restore operation..." + if ! "${PARTITIONER_TOOL}" -o 3; then + log "WARNING: Partition restore command failed. Check fabricmanager logs." + else + log "Partition restore operation completed." + fi + else + log "WARNING: Partitioner tool not found at ${PARTITIONER_TOOL}. Skipping restore." + fi + else + log "INFO: Started in normal mode, skipping partition restore operation." + fi fi # 4. Wait for the Fabric Manager daemon to exit.