diff --git a/pyproject.toml b/pyproject.toml index bdfecd610..9b08348ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -546,18 +546,8 @@ repro = [ # Resource Module - Resource monitoring # Use: pip install scitex[resource] -resource = [ - "psutil", - "matplotlib", - "pyyaml", - "joblib", - "ruamel.yaml", - "h5py", - "scipy", - "readchar", - # # Heavy dependencies handled by _AVAILABLE flags - # "torch", -] +# Real implementation lives in the standalone scitex-resource package. +resource = ["scitex-resource[sh]>=0.1.0"] # Reproduce Module - Reproducibility tools # Use: pip install scitex[reproduce] diff --git a/src/scitex/resource/README.md b/src/scitex/resource/README.md deleted file mode 100755 index 3cc539a15..000000000 --- a/src/scitex/resource/README.md +++ /dev/null @@ -1,81 +0,0 @@ - - - -# [`scitex.resource`](https://github.com/ywatanabe1989/scitex/tree/main/src/scitex/resource/) - -## Overview -The `scitex.resource` module provides comprehensive system resource monitoring and information gathering utilities. It offers an easy-to-use API for collecting detailed information about various system components, including CPU, memory, GPU, disk, and network. - -## Installation -```bash -pip install scitex -``` - -## Features -- Comprehensive system information gathering -- Easy-to-use API for resource monitoring -- Support for CPU, memory, GPU, disk, and network information -- Output in easily readable and parsable formats (YAML, JSON) - -## Quick Start -```python -import scitex - -# Gather system information -info = scitex.resource.gather_info() - -# Save the information to a file -scitex.io.save(info, "system_info.yaml") - -# Print specific information -print(f"CPU Usage: {info['cpu']['usage']}%") -print(f"Total RAM: {info['memory']['total']} GB") -print(f"GPU Name: {info['gpu'][0]['name']}") - -# Monitor system resources over time -for _ in range(10): - cpu_usage = scitex.resource.get_cpu_usage() - mem_usage = scitex.resource.get_memory_usage() - print(f"CPU: {cpu_usage}%, Memory: {mem_usage}%") - time.sleep(1) -``` - -## API Reference -- `scitex.resource.gather_info()`: Collects comprehensive system resource information -- `scitex.resource.get_cpu_info()`: Returns detailed CPU information -- `scitex.resource.get_memory_info()`: Returns memory usage statistics -- `scitex.resource.get_gpu_info()`: Returns GPU information (if available) -- `scitex.resource.get_disk_info()`: Returns disk usage and I/O statistics -- `scitex.resource.get_network_info()`: Returns network interface information -- `scitex.resource.get_cpu_usage()`: Returns current CPU usage percentage -- `scitex.resource.get_memory_usage()`: Returns current memory usage percentage - -## Example Output -The `gather_info()` function returns a dictionary containing detailed system information. For a full example of the output, please refer to: -https://github.com/ywatanabe1989/scitex/tree/main/src/scitex/res/_gather_info/info.yaml - -## Use Cases -- System monitoring and diagnostics -- Performance benchmarking -- Resource usage analysis -- Debugging hardware-related issues -- Generating system reports -- Automated system health checks - -## Performance -The `scitex.resource` module is designed to be lightweight and efficient, with minimal impact on system performance during monitoring. - -## Contributing -Contributions to improve `scitex.resource` are welcome. Please submit pull requests or open issues on the GitHub repository. - -## License -This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details. - -## Contact -Yusuke Watanabe (ywatanabe@scitex.ai) - -For more information and updates, please visit the [scitex GitHub repository](https://github.com/ywatanabe1989/scitex). diff --git a/src/scitex/resource/__init__.py b/src/scitex/resource/__init__.py index aae3e2886..353e78e4b 100755 --- a/src/scitex/resource/__init__.py +++ b/src/scitex/resource/__init__.py @@ -1,31 +1,20 @@ -#!/usr/bin/env python3 -"""Scitex resource module.""" +"""SciTeX resource — thin compatibility shim for scitex-resource. -from ._get_processor_usages import get_processor_usages -from ._get_specs import ( - _cpu_info, - _disk_info, - _memory_info, - _network_info, - _supple_nvidia_info, - _supple_os_info, - _supple_python_info, - _system_info, - get_specs, -) -from ._log_processor_usages import log_processor_usages, main +Aliases ``scitex.resource`` to the standalone ``scitex_resource`` package via +``sys.modules``. ``scitex.resource is scitex_resource``. -__all__ = [ - "get_processor_usages", - "get_specs", - "log_processor_usages", - "main", - "_cpu_info", - "_disk_info", - "_memory_info", - "_network_info", - "_supple_nvidia_info", - "_supple_os_info", - "_supple_python_info", - "_system_info", -] +Install: ``pip install scitex[resource]`` (or ``pip install scitex-resource``). +See: https://github.com/ywatanabe1989/scitex-resource +""" + +import sys as _sys + +try: + import scitex_resource as _real +except ImportError as _e: # pragma: no cover + raise ImportError( + "scitex.resource requires the 'scitex-resource' package. " + "Install with: pip install scitex[resource] (or: pip install scitex-resource)" + ) from _e + +_sys.modules[__name__] = _real diff --git a/src/scitex/resource/_get_processor_usages.py b/src/scitex/resource/_get_processor_usages.py deleted file mode 100755 index 9dc43acb1..000000000 --- a/src/scitex/resource/_get_processor_usages.py +++ /dev/null @@ -1,283 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Time-stamp: "2024-11-04 16:12:50 (ywatanabe)" -# File: ./scitex_repo/src/scitex/resource/_get_processor_usages.py - -""" -Functionality: - * Monitors and records system resource utilization (CPU, RAM, GPU, VRAM) -Input: - * None (uses system calls and psutil library) -Output: - * DataFrame containing resource usage statistics -Prerequisites: - * NVIDIA GPU with nvidia-smi installed - * psutil package -""" - -import os -import subprocess -import sys -from datetime import datetime -from typing import Optional, Tuple - -import matplotlib.pyplot as plt -import pandas as pd -import psutil - - -def get_processor_usages() -> pd.DataFrame: - """Gets current system resource usage statistics. - - Returns - ------- - pd.DataFrame - Resource usage data with columns: - - Timestamp: Timestamp - - CPU [%]: CPU utilization - - RAM [GiB]: RAM usage - - GPU [%]: GPU utilization - - VRAM [GiB]: VRAM usage - - Example - ------- - >>> df = get_proccessor_usages() - >>> print(df) - Timestamp CPU [%] RAM [GiB] GPU [%] VRAM [GiB] - 0 2024-11-04 10:30:15 25.3 8.2 65.0 4.5 - """ - try: - cpu_perc, ram_gb = _get_cpu_usage() - gpu_perc, vram_gb = _get_gpu_usage() - - sr = pd.Series( - { - "Timestamp": datetime.now(), - "CPU [%]": cpu_perc, - "RAM [GiB]": ram_gb, - "GPU [%]": gpu_perc, - "VRAM [GiB]": vram_gb, - } - ) - - return pd.DataFrame(sr).round(1).T - except Exception as err: - raise RuntimeError(f"Failed to get resource usage: {err}") - - -def _get_cpu_usage( - process: Optional[int] = os.getpid(), n_round: int = 1 -) -> Tuple[float, float]: - """Gets CPU and RAM usage statistics. - - Parameters - ---------- - process : int, optional - Process ID to monitor - n_round : int, optional - Number of decimal places to round to - - Returns - ------- - Tuple[float, float] - CPU usage percentage and RAM usage in GiB - """ - try: - cpu_usage_perc = psutil.cpu_percent() - ram_usage_gb = ( - psutil.virtual_memory().percent - / 100 - * psutil.virtual_memory().total - / (1024**3) - ) - return round(cpu_usage_perc, n_round), round(ram_usage_gb, n_round) - except Exception as err: - raise RuntimeError(f"Failed to get CPU/RAM usage: {err}") - - -def _get_gpu_usage(n_round: int = 1) -> Tuple[float, float]: - """Gets GPU and VRAM usage statistics. - - Parameters - ---------- - n_round : int, optional - Number of decimal places to round to - - Returns - ------- - Tuple[float, float] - GPU usage percentage and VRAM usage in GiB - """ - try: - result = subprocess.run( - [ - "nvidia-smi", - "--query-gpu=utilization.gpu,memory.used", - "--format=csv,nounits,noheader", - ], - capture_output=True, - text=True, - check=True, - ) - gpu_usage_perc, vram_usage_mib = result.stdout.strip().split(",") - vram_usage_gb = float(vram_usage_mib) / 1024 - return round(float(gpu_usage_perc), n_round), round(vram_usage_gb, n_round) - except: - return 0.0, 0.0 - # except subprocess.CalledProcessError as err: - # raise RuntimeError(f"Failed to execute nvidia-smi: {err}") - # except Exception as err: - # raise RuntimeError(f"Failed to get GPU/VRAM usage: {err}") - - -# def _get_gpu_usage(n_round: int = 1) -> Tuple[float, float]: -# """Gets GPU and VRAM usage statistics. - -# Parameters -# ---------- -# n_round : int, optional -# Number of decimal places to round to - -# Returns -# ------- -# Tuple[float, float] -# GPU usage percentage and VRAM usage in GiB -# """ -# try: -# result = subprocess.run( -# [ -# "nvidia-smi", -# "--query-gpu=utilization.gpu,memory.used", -# "--format=csv,nounits,noheader", -# ], -# capture_output=True, -# text=True, -# check=True, -# ) -# gpu_usage_perc, vram_usage_mib = result.stdout.strip().split(",") -# vram_usage_gb = float(vram_usage_mib) / 1024 -# return round(float(gpu_usage_perc), n_round), round(vram_usage_gb, n_round) -# except Exception as e: -# print(e) -# return 0.0, 0.0 # Return zeros when nvidia-smi is not available - - -if __name__ == "__main__": - import scitex - - CONFIG, sys.stdout, sys.stderr, plt, CC = scitex.session.start( - sys, plt, verbose=False - ) - - usage = scitex.resource.get_processor_usages() - scitex.io.save(usage, "usage.csv") - - scitex.session.close(CONFIG, verbose=False, notify=False) - -# EOF - -# #!/usr/bin/env python3 -# # -*- coding: utf-8 -*- -# # Time-stamp: "2024-11-04 10:27:35 (ywatanabe)" -# # File: ./scitex_repo/src/scitex/resource/_get_processor_usages.py - -# """ -# This script does XYZ. -# """ - -# # Functions -# import os -# import subprocess -# import sys -# from datetime import datetime - -# import matplotlib.pyplot as plt -# import scitex -# import pandas as pd -# import psutil - - -# # Functions -# def get_processor_usages(): -# """ -# Retrieves the current usage statistics for the CPU, RAM, GPU, and VRAM. - -# This function fetches the current usage percentages for the CPU and GPU, as well as the current usage in GiB for RAM and VRAM. -# The data is then compiled into a pandas DataFrame with the current timestamp. - -# Returns: -# pd.DataFrame: A pandas DataFrame containing the current usage statistics with the following columns: -# - Time: The timestamp when the data was retrieved. -# - CPU [%]: The CPU usage percentage. -# - RAM [GiB]: The RAM usage in GiB. -# - GPU [%]: The GPU usage percentage. -# - VRAM [GiB]: The VRAM usage in GiB. -# Each row in the DataFrame represents a single instance of data retrieval, rounded to 1 decimal place. - -# Example: -# >>> usage_df = get_processor_usages() -# >>> print(usage_df) -# """ -# cpu_perc, ram_gb = _get_cpu_usage() -# gpu_perc, vram_gb = _get_gpu_usage() - -# sr = pd.Series( -# { -# "Time": datetime.now(), -# "CPU [%]": cpu_perc, -# "RAM [GiB]": ram_gb, -# "GPU [%]": gpu_perc, -# "VRAM [GiB]": vram_gb, -# } -# ) - -# df = pd.DataFrame(sr).round(1).T - -# return df - - -# def _get_cpu_usage(process=os.getpid(), n_round=1): -# cpu_usage_perc = psutil.cpu_percent() -# ram_usage_gb = ( -# psutil.virtual_memory().percent -# / 100 -# * psutil.virtual_memory().total -# / (1024**3) -# ) -# return round(cpu_usage_perc, n_round), round(ram_usage_gb, n_round) - - -# def _get_gpu_usage(n_round=1): -# result = subprocess.run( -# [ -# "nvidia-smi", -# "--query-gpu=utilization.gpu,memory.used", -# "--format=csv,nounits,noheader", -# ], -# capture_output=True, -# text=True, -# ) -# gpu_usage_perc, _vram_usage_mib = result.stdout.strip().split(",") -# vram_usage_gb = float(_vram_usage_mib) / 1024 -# return round(float(gpu_usage_perc), n_round), round( -# float(vram_usage_gb), n_round -# ) - - -# # (YOUR AWESOME CODE) - -# if __name__ == "__main__": -# # Start -# CONFIG, sys.stdout, sys.stderr, plt, CC = scitex.session.start( -# sys, plt, verbose=False -# ) - -# usage = scitex.resource.get_processor_usages() -# scitex.io.save(usage, "usage.csv") - -# # Close -# scitex.session.close(CONFIG, verbose=False, notify=False) - -# - -# EOF diff --git a/src/scitex/resource/_get_specs.py b/src/scitex/resource/_get_specs.py deleted file mode 100755 index e2cdddcd3..000000000 --- a/src/scitex/resource/_get_specs.py +++ /dev/null @@ -1,282 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Time-stamp: "2024-11-04 14:16:49 (ywatanabe)" -# File: ./scitex_repo/src/scitex/resource/_get_specs.py - -""" -This script provides detailed system information including system basics, boot time, CPU, memory, disk, network, and custom user environment variables. -""" - -import platform as _platform -import sys -from datetime import datetime as _datetime -from pprint import pprint - -import matplotlib.pyplot as plt -import psutil as _psutil -import yaml as _yaml - -from scitex.str import readable_bytes - -from ._utils._get_env_info import get_env_info - - -def get_specs( - system=True, - # boot_time=True, - cpu=True, - gpu=True, - disk=True, - network=True, - verbose=False, - yaml=False, -): - """ - Collects and returns system specifications including system information, CPU, GPU, disk, and network details. - - This function gathers various pieces of system information based on the parameters provided. It can return the data in a dictionary format or print it out based on the verbose flag. Additionally, there's an option to format the output as YAML. - - Arguments: - system (bool): If True, collects system-wide information such as OS and node name. Default is True. - boot_time (bool): If True, collects system boot time. Currently commented out in the implementation. Default is True. - cpu (bool): If True, collects CPU-specific information including frequency and usage. Default is True. - gpu (bool): If True, collects GPU-specific information. Default is True. - disk (bool): If True, collects disk usage information for all partitions. Default is True. - network (bool): If True, collects network interface and traffic information. Default is True. - verbose (bool): If True, prints the collected information using pprint. Default is False. - yaml (bool): If True, formats the collected information as YAML. This modifies the return type to a YAML formatted string. Default is False. - - Returns: - dict or str: By default, returns a dictionary containing the collected system specifications. If `yaml` is True, returns a YAML-formatted string instead. - - Note: - - The actual collection of system, CPU, GPU, disk, and network information depends on the availability of corresponding libraries and access permissions. - - The `boot_time` argument is currently not used as its corresponding code is commented out. - - The function uses global variables and imports within its scope, which might affect its reusability and testability. - - Example: - >>> specs = get_specs(verbose=True) - This will print and return the system specifications based on the default parameters. - - Dependencies: - - This function depends on the `scitex` library for accessing system information and formatting output. Ensure this library is installed and properly configured. - - Python standard libraries: `datetime`, `platform`, `psutil`, `yaml` (optional for YAML output). - - Raises: - PermissionError: If the function lacks necessary permissions to access certain system information, especially disk and network details. - """ - - # To prevent import errors, _SUPPLE_INFO is collected here. - global _SUPPLE_INFO - _SUPPLE_INFO = get_env_info()._asdict() - - collected_info = {} # OrderedDict() - - collected_info["Collected Time"] = _datetime.now().strftime("%Y-%m-%d %H:%M:%S") - if system: - collected_info["System Information"] = _system_info() - # if boot_time: - # collected_info["Boot Time"] = _boot_time_info() - if cpu: - collected_info["CPU Info"] = _cpu_info() - collected_info["Memory Info"] = _memory_info() - if gpu: - collected_info["GPU Info"] = _supple_nvidia_info() - # scitex.gen.placeholder() - if disk: - collected_info["Disk Info"] = _disk_info() - if network: - collected_info["Network Info"] = _network_info() - - if yaml: - collected_info = _yaml.dump(collected_info, sort_keys=False) - - if verbose: - pprint(collected_info) - - return collected_info - - -def _system_info(): - uname = _platform.uname() - return { - "OS": _supple_os_info()["os"], - # "GCC version": _supple_os_info()["gcc_version"], - # "System": uname.system, - "Node Name": uname.node, - "Release": uname.release, - "Version": uname.version, - # "Machine": uname.machine, - # "Processor": uname.processor, - } - - -# def _boot_time_info(): -# boot_time_timestamp = _psutil.boot_time() -# bt = _datetime.fromtimestamp(boot_time_timestamp) -# return { -# "Boot Time": f"{bt.year}-{bt.month:02d}-{bt.day:02d} {bt.hour:02d}:{bt.minute:02d}:{bt.second:02d}" -# } - - -def _cpu_info(): - cpufreq = _psutil.cpu_freq() - cpu_usage_per_core = _psutil.cpu_percent(percpu=True, interval=1) - return { - "Physical cores": _psutil.cpu_count(logical=False), - "Total cores": _psutil.cpu_count(logical=True), - "Max Frequency": f"{cpufreq.max:.2f} MHz", - "Min Frequency": f"{cpufreq.min:.2f} MHz", - "Current Frequency": f"{cpufreq.current:.2f} MHz", - "CPU Usage Per Core": { - f"Core {i}": f"{percentage}%" - for i, percentage in enumerate(cpu_usage_per_core) - }, - "Total CPU Usage": f"{_psutil.cpu_percent()}%", - } - - -def _memory_info(): - import scitex - - svmem = _psutil.virtual_memory() - swap = _psutil.swap_memory() - - return { - "Memory": { - "Total": readable_bytes(svmem.total), - "Available": readable_bytes(svmem.available), - "Used": readable_bytes(svmem.used), - "Percentage": svmem.percent, - }, - "SWAP": { - "Total": readable_bytes(swap.total), - "Free": readable_bytes(swap.free), - "Used": readable_bytes(swap.used), - "Percentage": swap.percent, - }, - } - - -def _disk_info(): - import scitex - - partitions_info = {} - partitions = _psutil.disk_partitions() - for partition in partitions: - try: - usage = _psutil.disk_usage(partition.mountpoint) - partitions_info[partition.device] = { - "Mountpoint": partition.mountpoint, - "File system type": partition.fstype, - "Total Size": readable_bytes(usage.total), - "Used": readable_bytes(usage.used), - "Free": readable_bytes(usage.free), - "Percentage": usage.percent, - } - except PermissionError: - continue - - disk_io = _psutil.disk_io_counters() - return { - "Partitions": partitions_info, - "Total read": readable_bytes(disk_io.read_bytes), - "Total write": readable_bytes(disk_io.write_bytes), - } - - -def _network_info(): - import scitex - - if_addrs = _psutil.net_if_addrs() - interfaces = {} - for interface_name, interface_addresses in if_addrs.items(): - interface_info = [] - for address in interface_addresses: - interface_info.append( - { - # "Address Type": "IP" if address.family == _psutil.AF_INET else "MAC", - "Address": address.address, - "Netmask": address.netmask, - "Broadcast": address.broadcast, - } - ) - interfaces[interface_name] = interface_info - - net_io = _psutil.net_io_counters() - return { - "Interfaces": interfaces, - "Total Sent": readable_bytes(net_io.bytes_sent), - "Total Received": readable_bytes(net_io.bytes_recv), - } - - -def _python_info(): - return _supple_python_info() - - -def _supple_os_info(): - _SUPPLE_OS_KEYS = [ - "os", - "gcc_version", - ] - return {k: _SUPPLE_INFO[k] for k in _SUPPLE_OS_KEYS} - - -def _supple_python_info(): - _SUPPLE_PYTHON_KEYS = [ - "python_version", - "torch_version", - "is_cuda_available", - "pip_version", - "pip_packages", - "conda_packages", - ] - - return {k: _SUPPLE_INFO[k] for k in _SUPPLE_PYTHON_KEYS} - - -def _supple_nvidia_info(): - _SUPPLE_NVIDIA_KEYS = [ - "nvidia_gpu_models", - "nvidia_driver_version", - "cuda_runtime_version", - "cudnn_version", - ] - - def replace_key(key): - return key - - def replace_key(key): - key = key.replace("_", " ") - key = key.replace("nvidia", "NVIDIA") - key = key.replace("gpu", "GPU") - key = key.replace("cuda", "CUDA") - key = key.replace("cudnn", "cuDNN") - key = key.replace("driver", "Driver") - key = key.replace("runtime", "Runtime") - return key - - return {replace_key(k): _SUPPLE_INFO[k] for k in _SUPPLE_NVIDIA_KEYS} - - -if __name__ == "__main__": - import scitex - - # Start - CONFIG, sys.stdout, sys.stderr, plt, CC = scitex.session.start(sys, plt) - - info = scitex.res.get_specs() - scitex.io.save(info, "specs.yaml") - - # Close - scitex.session.close(CONFIG) - -# EOF - -""" -/home/ywatanabe/proj/entrance/scitex/res/_get_specs.py -""" - - -# EOF diff --git a/src/scitex/resource/_get_specs/info.yaml b/src/scitex/resource/_get_specs/info.yaml deleted file mode 100755 index 77b947852..000000000 --- a/src/scitex/resource/_get_specs/info.yaml +++ /dev/null @@ -1,122 +0,0 @@ -Collected Time: '2024-04-13 18:05:17' -System Information: - OS: Rocky Linux 9.3 (Blue Onyx) (x86_64) - Node Name: '444' - Release: 5.14.0-362.24.1.el9_3.x86_64 - Version: '#1 SMP PREEMPT_DYNAMIC Wed Mar 13 17:33:16 UTC 2024' -CPU Info: - Physical cores: 16 - Total cores: 32 - Max Frequency: 4500.00 MHz - Min Frequency: 3000.00 MHz - Current Frequency: 4530.39 MHz - CPU Usage Per Core: - Core 0: 0.0% - Core 1: 0.0% - Core 2: 0.0% - Core 3: 0.0% - Core 4: 0.0% - Core 5: 0.0% - Core 6: 0.0% - Core 7: 0.0% - Core 8: 0.0% - Core 9: 0.0% - Core 10: 0.0% - Core 11: 0.0% - Core 12: 0.0% - Core 13: 0.0% - Core 14: 0.0% - Core 15: 0.0% - Core 16: 0.0% - Core 17: 0.0% - Core 18: 0.0% - Core 19: 0.0% - Core 20: 0.0% - Core 21: 0.0% - Core 22: 0.0% - Core 23: 0.0% - Core 24: 0.0% - Core 25: 0.0% - Core 26: 0.0% - Core 27: 0.0% - Core 28: 0.0% - Core 29: 0.0% - Core 30: 0.0% - Core 31: 0.0% - Total CPU Usage: 3.0% -Memory Info: - Memory: - Total: 61.7GiB - Available: 58.7GiB - Used: 2.1GiB - Percentage: 4.8 - SWAP: - Total: 31.0GiB - Free: 30.7GiB - Used: 337.4MiB - Percentage: 1.1 -GPU Info: - NVIDIA GPU models: 'GPU 0: NVIDIA GeForce RTX 4090' - NVIDIA Driver version: '550.67' - CUDA Runtime version: - cuDNN version: -Disk Info: - Partitions: - /dev/mapper/rl-root: - Mountpoint: / - File system type: xfs - Total Size: 70.0GiB - Used: 15.0GiB - Free: 55.0GiB - Percentage: 21.4 - /dev/mapper/rl-home: - Mountpoint: /home - File system type: xfs - Total Size: 850.8GiB - Used: 189.4GiB - Free: 661.5GiB - Percentage: 22.3 - /dev/nvme0n1p2: - Mountpoint: /boot - File system type: xfs - Total Size: 1014.0MiB - Used: 1003.3MiB - Free: 10.7MiB - Percentage: 98.9 - /dev/nvme0n1p1: - Mountpoint: /boot/efi - File system type: vfat - Total Size: 598.8MiB - Used: 7.0MiB - Free: 591.8MiB - Percentage: 1.2 - Total read: 244.6GiB - Total write: 465.2GiB -Network Info: - Interfaces: - lo: - - Address: 127.0.0.1 - Netmask: 255.0.0.0 - Broadcast: - - Address: ::1 - Netmask: ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff - Broadcast: - - Address: 00:00:00:00:00:00 - Netmask: - Broadcast: - enp24s0: - - Address: 172.16.31.31 - Netmask: 255.255.255.0 - Broadcast: 172.16.31.255 - - Address: fe80::6bbe:a317:c388:7a0b%enp24s0 - Netmask: 'ffff:ffff:ffff:ffff::' - Broadcast: - - Address: 04:7c:16:6d:e2:0a - Netmask: - Broadcast: ff:ff:ff:ff:ff:ff - wlp31s0: - - Address: e6:6c:a0:a3:52:f7 - Netmask: - Broadcast: ff:ff:ff:ff:ff:ff - Total Sent: 3.7GiB - Total Received: 4.2GiB diff --git a/src/scitex/resource/_get_specs/specs.yaml b/src/scitex/resource/_get_specs/specs.yaml deleted file mode 100755 index 195b39e2a..000000000 --- a/src/scitex/resource/_get_specs/specs.yaml +++ /dev/null @@ -1,129 +0,0 @@ -Collected Time: '2024-04-21 23:09:27' -System Information: - OS: Rocky Linux 9.3 (Blue Onyx) (x86_64) - Node Name: '444' - Release: 5.14.0-362.24.1.el9_3.x86_64 - Version: '#1 SMP PREEMPT_DYNAMIC Wed Mar 13 17:33:16 UTC 2024' -CPU Info: - Physical cores: 16 - Total cores: 32 - Max Frequency: 4500.00 MHz - Min Frequency: 3000.00 MHz - Current Frequency: 4531.93 MHz - CPU Usage Per Core: - Core 0: 0.0% - Core 1: 0.0% - Core 2: 0.0% - Core 3: 0.0% - Core 4: 0.0% - Core 5: 0.0% - Core 6: 0.0% - Core 7: 0.0% - Core 8: 0.0% - Core 9: 0.0% - Core 10: 0.0% - Core 11: 0.0% - Core 12: 0.0% - Core 13: 0.0% - Core 14: 0.0% - Core 15: 0.0% - Core 16: 0.0% - Core 17: 0.0% - Core 18: 0.0% - Core 19: 0.0% - Core 20: 0.0% - Core 21: 5.9% - Core 22: 0.0% - Core 23: 0.0% - Core 24: 0.0% - Core 25: 0.0% - Core 26: 0.0% - Core 27: 0.0% - Core 28: 1.0% - Core 29: 0.0% - Core 30: 1.0% - Core 31: 0.0% - Total CPU Usage: 3.2% -Memory Info: - Memory: - Total: 61.7GiB - Available: 58.5GiB - Used: 2.4GiB - Percentage: 5.2 - SWAP: - Total: 31.0GiB - Free: 30.5GiB - Used: 599.9MiB - Percentage: 1.9 -GPU Info: - NVIDIA GPU models: 'GPU 0: NVIDIA GeForce RTX 4090' - NVIDIA Driver version: '550.67' - CUDA Runtime version: - cuDNN version: -Disk Info: - Partitions: - /dev/mapper/rl-root: - Mountpoint: / - File system type: xfs - Total Size: 70.0GiB - Used: 14.8GiB - Free: 55.1GiB - Percentage: 21.2 - /dev/mapper/rl-home: - Mountpoint: /home - File system type: xfs - Total Size: 850.8GiB - Used: 211.1GiB - Free: 639.8GiB - Percentage: 24.8 - /dev/nvme0n1p2: - Mountpoint: /boot - File system type: xfs - Total Size: 1014.0MiB - Used: 1003.3MiB - Free: 10.7MiB - Percentage: 98.9 - /dev/nvme0n1p1: - Mountpoint: /boot/efi - File system type: vfat - Total Size: 598.8MiB - Used: 7.0MiB - Free: 591.8MiB - Percentage: 1.2 - /dev/hptblock0n0p: - Mountpoint: /mnt/ssd - File system type: ext4 - Total Size: 7.2TiB - Used: 6.7TiB - Free: 122.6GiB - Percentage: 98.3 - Total read: 5.6TiB - Total write: 13.6TiB -Network Info: - Interfaces: - lo: - - Address: 127.0.0.1 - Netmask: 255.0.0.0 - Broadcast: - - Address: ::1 - Netmask: ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff - Broadcast: - - Address: 00:00:00:00:00:00 - Netmask: - Broadcast: - enp24s0: - - Address: 172.16.31.31 - Netmask: 255.255.255.0 - Broadcast: 172.16.31.255 - - Address: fe80::6bbe:a317:c388:7a0b%enp24s0 - Netmask: 'ffff:ffff:ffff:ffff::' - Broadcast: - - Address: 04:7c:16:6d:e2:0a - Netmask: - Broadcast: ff:ff:ff:ff:ff:ff - wlp31s0: - - Address: 92:5f:91:42:08:d6 - Netmask: - Broadcast: ff:ff:ff:ff:ff:ff - Total Sent: 11.5GiB - Total Received: 15.5GiB diff --git a/src/scitex/resource/_log_processor_usages.py b/src/scitex/resource/_log_processor_usages.py deleted file mode 100755 index 26a74bb0f..000000000 --- a/src/scitex/resource/_log_processor_usages.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Time-stamp: "2024-11-04 16:28:53 (ywatanabe)" -# File: ./scitex_repo/src/scitex/resource/_log_processor_usages.py - -""" -Functionality: - * Monitors and logs system resource utilization over time -Input: - * Path for saving logs - * Monitoring duration and interval -Output: - * CSV file containing time-series resource usage data -Prerequisites: - * scitex package with processor usage monitoring capabilities -""" - -"""Imports""" -import math -import os -import sys -import time -from multiprocessing import Process -from typing import Union - -import matplotlib.pyplot as plt -import pandas as pd - -import scitex -from scitex.io._load import load -from scitex.io._save import save -from scitex.sh import sh -from scitex.str import printc - -from ._get_processor_usages import get_processor_usages - -"""Functions & Classes""" - - -def log_processor_usages( - path: str = "/tmp/scitex/processor_usages.csv", - limit_min: float = 30, - interval_s: float = 1, - init: bool = True, - verbose: bool = False, - background: bool = False, -) -> Union[None, Process]: - """Logs system resource usage over time. - - Parameters - ---------- - path : str - Path to save the log file - limit_min : float - Monitoring duration in minutes - interval_s : float - Sampling interval in seconds - init : bool - Whether to clear existing log file - verbose : bool - Whether to print the log - background : bool - Whether to run in background - - Returns - ------- - Union[None, Process] - Process object if background=True, None otherwise - """ - if background: - process = Process( - target=_log_processor_usages, - args=(path, limit_min, interval_s, init, verbose), - ) - process.start() - return process - - return _log_processor_usages( - path=path, - limit_min=limit_min, - interval_s=interval_s, - init=init, - verbose=verbose, - ) - - -def _log_processor_usages( - path: str = "/tmp/scitex/processor_usages.csv", - limit_min: float = 30, - interval_s: float = 1, - init: bool = True, - verbose: bool = False, -) -> None: - """Logs system resource usage over time. - - Parameters - ---------- - path : str - Path to save the log file - limit_min : float - Monitoring duration in minutes - interval_s : float - Sampling interval in seconds - init : bool - Whether to clear existing log file - verbose : bool - Whether to print the log - - Example - ------- - >>> log_processor_usages(path="usage_log.csv", limit_min=5) - """ - assert path.endswith(".csv"), "Path must end with .csv" - - # Log file initialization - _ensure_log_file(path, init) - printc(f"Log file can be monitored with with `tail -f {path}`") - - limit_s = limit_min * 60 - n_max = math.ceil(limit_s // interval_s) - - for _ in range(n_max): - _add(path, verbose=verbose) - time.sleep(interval_s) - - -# def _ensure_log_file(path: str, init: bool) -> None: -# def _create_path(path): -# os.makedirs(os.path.dirname(path), exist_ok=True) -# empty_df = pd.DataFrame() -# save(empty_df, path, verbose=False) -# printc(f"{path} created.") - -# if not os.path.exists(path): -# _create_path(path) - -# else: -# if init and os.path.exists(path): -# try: -# sh(f"rm -f {path}") -# _create_path(path) -# except Exception as err: -# raise RuntimeError(f"Failed to init log file: {err}") - -# def _add(path: str, verbose: bool = True) -> None: -# past = load(path) -# now = get_processor_usages() - -# combined = pd.concat([past, now]).round(3) -# save(combined, path, verbose=verbose) - - -def _add(path: str, verbose: bool = True) -> None: - """Appends current resource usage to CSV file.""" - now = get_processor_usages() - - # Append mode without loading entire file - with open(path, "a") as f: - now.to_csv(f, header=f.tell() == 0, index=False) - - -def _ensure_log_file(path: str, init: bool) -> None: - """Creates or reinitializes log file with headers.""" - - def _create_path(path): - os.makedirs(os.path.dirname(path), exist_ok=True) - # Write only headers - headers = ["Timestamp", "CPU [%]", "RAM [GiB]", "GPU [%]", "VRAM [GiB]"] - pd.DataFrame(columns=headers).to_csv(path, index=False) - printc(f"{path} created.") - - if not os.path.exists(path): - _create_path(path) - elif init: - try: - sh(f"rm -f {path}") - _create_path(path) - except Exception as err: - raise RuntimeError(f"Failed to init log file: {err}") - - -main = log_processor_usages - -if __name__ == "__main__": - CONFIG, sys.stdout, sys.stderr, plt, CC = scitex.session.start( - sys, plt, verbose=False - ) - main() - scitex.session.close(CONFIG, verbose=False, notify=False) - -# python -c "import scitex; scitex.resource.log_processor_usages(\"/tmp/processor_usages.csv\", init=True)" - -# EOF diff --git a/src/scitex/resource/_skills/SKILL.md b/src/scitex/resource/_skills/SKILL.md deleted file mode 100644 index 84546387b..000000000 --- a/src/scitex/resource/_skills/SKILL.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -name: stx.resource -description: System resource monitoring — one-shot CPU/RAM/GPU snapshots, time-series logging, and full hardware spec collection. ---- - -# stx.resource — Skills Index - -Monitor system resources during scientific computations. Covers real-time usage snapshots, CSV time-series logging, and complete hardware/software specification collection. - -## Sub-skills - -| File | Description | -|------|-------------| -| [monitor.md](monitor.md) | get_processor_usages (CPU/RAM/GPU/VRAM DataFrame), log_processor_usages (CSV logging, background process) | -| [specs.md](specs.md) | get_specs (full dict or YAML), component helpers (_cpu_info, _memory_info, etc.) | - -## Quick Reference - -```python -from scitex.resource import get_processor_usages, get_specs, log_processor_usages - -df = get_processor_usages() # One-row DataFrame: CPU/RAM/GPU/VRAM -specs = get_specs(yaml=True) # Full system specs as YAML string -log_processor_usages( - "/tmp/usage.csv", limit_min=10, interval_s=1, background=True -) -``` diff --git a/src/scitex/resource/_skills/monitor.md b/src/scitex/resource/_skills/monitor.md deleted file mode 100644 index 9bcb3187d..000000000 --- a/src/scitex/resource/_skills/monitor.md +++ /dev/null @@ -1,60 +0,0 @@ ---- -description: Get a one-shot snapshot of CPU/RAM/GPU/VRAM usage or log it over time to a CSV file. ---- - -# stx.resource — Processor Usage Monitoring - -## get_processor_usages - -Returns a one-row `pd.DataFrame` with current CPU, RAM, GPU, and VRAM readings. - -```python -from scitex.resource import get_processor_usages - -df = get_processor_usages() -print(df) -# Timestamp CPU [%] RAM [GiB] GPU [%] VRAM [GiB] -# 0 2024-11-04 10:30:15 25.3 8.2 65.0 4.5 -``` - -Columns: -- `Timestamp` — `datetime` of the sample -- `CPU [%]` — system-wide CPU utilization via `psutil.cpu_percent()` -- `RAM [GiB]` — RAM used in GiB (percent × total) -- `GPU [%]` — GPU utilization from `nvidia-smi`; `0.0` if unavailable -- `VRAM [GiB]` — VRAM used in GiB from `nvidia-smi`; `0.0` if unavailable - -GPU values fall back to `0.0` silently when `nvidia-smi` is not present. - -## log_processor_usages - -Polls `get_processor_usages()` at a fixed interval and appends each row to a CSV file. - -```python -from scitex.resource import log_processor_usages - -# Foreground: blocks for 30 minutes, sampling every 1 s -log_processor_usages( - path="/tmp/scitex/processor_usages.csv", - limit_min=30, - interval_s=1, - init=True, # True: clear existing file before starting - verbose=False, -) - -# Background: returns a multiprocessing.Process -proc = log_processor_usages( - path="/tmp/resource_log.csv", - limit_min=60, - interval_s=5, - background=True, -) -proc.terminate() # Stop early - -# Monitor live with tail (printed to console at start) -# tail -f /tmp/scitex/processor_usages.csv -``` - -The CSV is written in append mode without loading the whole file, keeping memory usage flat. - -`main` is an alias for `log_processor_usages` (used by `python -m scitex.resource`). diff --git a/src/scitex/resource/_skills/monitoring.md b/src/scitex/resource/_skills/monitoring.md deleted file mode 100644 index 52a82922c..000000000 --- a/src/scitex/resource/_skills/monitoring.md +++ /dev/null @@ -1,58 +0,0 @@ ---- -description: Sample instantaneous CPU/GPU/memory usage with get_processor_usages() and log usage over time to a CSV file with log_processor_usages(). ---- - -# Usage Monitoring - -## get_processor_usages - -Return a dict of current CPU, GPU, and memory utilization percentages. - -```python -get_processor_usages() -> dict -``` - -```python -import scitex as stx - -usage = stx.resource.get_processor_usages() -print(usage) -# { -# 'cpu_percent': 23.4, -# 'memory_percent': 61.2, -# 'gpu_0_percent': 87.0, -# 'gpu_0_memory_percent': 52.3, -# } -``` - ---- - -## log_processor_usages - -Poll `get_processor_usages()` at regular intervals and append rows to a CSV file. - -```python -log_processor_usages( - output_path: str = "resource_log.csv", - interval_s: float = 1.0, - duration_s: float | None = None, -) -> None -``` - -Runs until `duration_s` seconds elapse (or indefinitely if `None`). Writes one row per interval. - -```python -import scitex as stx -import multiprocessing - -# Log resource usage in the background while training -p = multiprocessing.Process( - target=stx.resource.log_processor_usages, - kwargs={"output_path": "training_resources.csv", "interval_s": 5.0}, -) -p.start() - -# ... run training ... - -p.terminate() -``` diff --git a/src/scitex/resource/_skills/specs.md b/src/scitex/resource/_skills/specs.md deleted file mode 100644 index 6e983ec0b..000000000 --- a/src/scitex/resource/_skills/specs.md +++ /dev/null @@ -1,65 +0,0 @@ ---- -description: Collect full system hardware and software specifications as a dict or YAML string. ---- - -# stx.resource — System Specifications - -## get_specs - -Collects system info into a nested dict (or YAML string). - -```python -from scitex.resource import get_specs - -specs = get_specs() -# { -# "Collected Time": "2024-11-04 10:30:00", -# "System Information": {"OS": "Linux", "Node Name": "myhost", ...}, -# "CPU Info": {"Physical cores": 8, "Total cores": 16, ...}, -# "Memory Info": {"Memory": {"Total": "32.0 GiB", ...}, "SWAP": {...}}, -# "GPU Info": {"NVIDIA GPU models": [...], "NVIDIA Driver Version": "..."}, -# "Disk Info": {"Partitions": {...}, "Total read": "...", "Total write": "..."}, -# "Network Info": {"Interfaces": {...}, "Total Sent": "...", "Total Received": "..."}, -# } - -# Select only some sections -cpu_only = get_specs(cpu=True, gpu=False, disk=False, network=False) - -# Print to console -get_specs(verbose=True) - -# Get as YAML string -yaml_str = get_specs(yaml=True) -print(yaml_str) - -# Save to file (using stx.io) -import scitex as stx -specs = get_specs() -stx.io.save(specs, "specs.yaml") -``` - -Parameters (all default `True`): -- `system` — OS, node name, kernel release/version -- `cpu` — core counts, frequency range, per-core and total usage; also includes memory info -- `gpu` — NVIDIA GPU models, driver version, CUDA runtime, cuDNN version (via `_supple_nvidia_info`) -- `disk` — partition mount points, sizes, IO counters -- `network` — interface addresses, total bytes sent/received -- `verbose` — print via `pprint` -- `yaml` — return YAML string instead of dict - -## Component-level helpers - -```python -from scitex.resource import ( - _cpu_info, _memory_info, _disk_info, _network_info, - _supple_nvidia_info, _supple_os_info, _supple_python_info, - _system_info, -) - -print(_cpu_info()) # dict with core counts, freq, per-core usage -print(_memory_info()) # dict with RAM and SWAP stats in readable units -print(_supple_nvidia_info()) # dict with GPU model, driver, CUDA, cuDNN -print(_supple_python_info()) # dict with python_version, torch_version, pip packages -``` - -All readable byte values use `scitex.str.readable_bytes` for human-friendly formatting (e.g., `"32.0 GiB"`). diff --git a/src/scitex/resource/_utils/__init__.py b/src/scitex/resource/_utils/__init__.py deleted file mode 100755 index ea2287d9a..000000000 --- a/src/scitex/resource/_utils/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Time-stamp: "2024-11-04 14:13:44 (ywatanabe)" -# File: ./scitex_repo/src/scitex/resource/_utils/__init__.py - -import importlib -import inspect -import os - -# Get the current directory -current_dir = os.path.dirname(__file__) - -# Iterate through all Python files in the current directory -for filename in os.listdir(current_dir): - if filename.endswith(".py") and not filename.startswith("__"): - module_name = filename[:-3] # Remove .py extension - module = importlib.import_module(f".{module_name}", package=__name__) - - # Import functions, classes, and specific module-level variables - for name, obj in inspect.getmembers(module): - if inspect.isfunction(obj) or inspect.isclass(obj): - if not name.startswith("_"): - globals()[name] = obj - # Also import specific module-level variables - elif name in ["TORCH_AVAILABLE", "env_info_fmt"]: - globals()[name] = obj - -# Clean up temporary variables -del os, importlib, inspect, current_dir, filename, module_name, module, name, obj - -# EOF diff --git a/src/scitex/resource/_utils/_get_env_info.py b/src/scitex/resource/_utils/_get_env_info.py deleted file mode 100755 index 69ed29a65..000000000 --- a/src/scitex/resource/_utils/_get_env_info.py +++ /dev/null @@ -1,469 +0,0 @@ -#!/usr/bin/env python3 - -# This script outputs relevant system environment info -# Run it with `python collect_env.py`. -import locale -import os -import re -import subprocess -import sys -from collections import namedtuple - -try: - import torch - - TORCH_AVAILABLE = True -except (ImportError, NameError, AttributeError, OSError): - TORCH_AVAILABLE = False - -# System Environment Information -SystemEnv = namedtuple( - "SystemEnv", - [ - "torch_version", - "is_debug_build", - "cuda_compiled_version", - "gcc_version", - "clang_version", - "cmake_version", - "os", - "python_version", - "is_cuda_available", - "cuda_runtime_version", - "nvidia_driver_version", - "nvidia_gpu_models", - "cudnn_version", - "pip_version", # 'pip' or 'pip3' - "pip_packages", - "conda_packages", - "hip_compiled_version", - "hip_runtime_version", - "miopen_runtime_version", - ], -) - - -def run(command): - """Returns (return-code, stdout, stderr)""" - p = subprocess.Popen( - command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True - ) - raw_output, raw_err = p.communicate() - rc = p.returncode - if get_platform() == "win32": - enc = "oem" - else: - enc = locale.getpreferredencoding() - output = raw_output.decode(enc) - err = raw_err.decode(enc) - return rc, output.strip(), err.strip() - - -def run_and_read_all(run_lambda, command): - """Runs command using run_lambda; reads and returns entire output if rc is 0""" - rc, out, _ = run_lambda(command) - if rc != 0: - return None - return out - - -def run_and_parse_first_match(run_lambda, command, regex): - """Runs command using run_lambda, returns the first regex match if it exists""" - rc, out, _ = run_lambda(command) - if rc != 0: - return None - match = re.search(regex, out) - if match is None: - return None - return match.group(1) - - -def get_conda_packages(run_lambda): - if get_platform() == "win32": - system_root = os.environ.get("SYSTEMROOT", "C:\\Windows") - findstr_cmd = os.path.join(system_root, "System32", "findstr") - grep_cmd = rf'{findstr_cmd} /R "torch numpy cudatoolkit soumith mkl magma"' - else: - grep_cmd = r'grep "torch\|numpy\|cudatoolkit\|soumith\|mkl\|magma"' - conda = os.environ.get("CONDA_EXE", "conda") - out = run_and_read_all(run_lambda, conda + " list | " + grep_cmd) - if out is None: - return out - # Comment starting at beginning of line - comment_regex = re.compile(r"^#.*\n") - return re.sub(comment_regex, "", out) - - -def get_gcc_version(run_lambda): - return run_and_parse_first_match(run_lambda, "gcc --version", r"gcc (.*)") - - -def get_clang_version(run_lambda): - return run_and_parse_first_match( - run_lambda, "clang --version", r"clang version (.*)" - ) - - -def get_cmake_version(run_lambda): - return run_and_parse_first_match(run_lambda, "cmake --version", r"cmake (.*)") - - -def get_nvidia_driver_version(run_lambda): - if get_platform() == "darwin": - cmd = "kextstat | grep -i cuda" - return run_and_parse_first_match( - run_lambda, cmd, r"com[.]nvidia[.]CUDA [(](.*?)[)]" - ) - smi = get_nvidia_smi() - return run_and_parse_first_match(run_lambda, smi, r"Driver Version: (.*?) ") - - -def get_gpu_info(run_lambda): - if get_platform() == "darwin" or ( - TORCH_AVAILABLE - and hasattr(torch.version, "hip") - and torch.version.hip is not None - ): - if TORCH_AVAILABLE and torch.cuda.is_available(): - return torch.cuda.get_device_name(None) - return None - smi = get_nvidia_smi() - uuid_regex = re.compile(r" \(UUID: .+?\)") - rc, out, _ = run_lambda(smi + " -L") - if rc != 0: - return None - # Anonymize GPUs by removing their UUID - return re.sub(uuid_regex, "", out) - - -def get_running_cuda_version(run_lambda): - return run_and_parse_first_match(run_lambda, "nvcc --version", r"release .+ V(.*)") - - -def get_cudnn_version(run_lambda): - """This will return a list of libcudnn.so; it's hard to tell which one is being used""" - if get_platform() == "win32": - system_root = os.environ.get("SYSTEMROOT", "C:\\Windows") - cuda_path = os.environ.get("CUDA_PATH", "%CUDA_PATH%") - where_cmd = os.path.join(system_root, "System32", "where") - cudnn_cmd = f'{where_cmd} /R "{cuda_path}\\bin" cudnn*.dll' - elif get_platform() == "darwin": - # CUDA libraries and drivers can be found in /usr/local/cuda/. See - # https://docs.nvidia.com/cuda/cuda-installation-guide-mac-os-x/index.html#install - # https://docs.nvidia.com/deeplearning/sdk/cudnn-install/index.html#installmac - # Use CUDNN_LIBRARY when cudnn library is installed elsewhere. - cudnn_cmd = "ls /usr/local/cuda/lib/libcudnn*" - else: - cudnn_cmd = 'ldconfig -p | grep libcudnn | rev | cut -d" " -f1 | rev' - rc, out, _ = run_lambda(cudnn_cmd) - # find will return 1 if there are permission errors or if not found - if len(out) == 0 or (rc != 1 and rc != 0): - l = os.environ.get("CUDNN_LIBRARY") - if l is not None and os.path.isfile(l): - return os.path.realpath(l) - return None - files_set = set() - for fn in out.split("\n"): - fn = os.path.realpath(fn) # eliminate symbolic links - if os.path.isfile(fn): - files_set.add(fn) - if not files_set: - return None - # Alphabetize the result because the order is non-deterministic otherwise - files = sorted(files_set) - if len(files) == 1: - return files[0] - result = "\n".join(files) - return f"Probably one of the following:\n{result}" - - -def get_nvidia_smi(): - # Note: nvidia-smi is currently available only on Windows and Linux - smi = "nvidia-smi" - if get_platform() == "win32": - system_root = os.environ.get("SYSTEMROOT", "C:\\Windows") - program_files_root = os.environ.get("PROGRAMFILES", "C:\\Program Files") - legacy_path = os.path.join( - program_files_root, "NVIDIA Corporation", "NVSMI", smi - ) - new_path = os.path.join(system_root, "System32", smi) - smis = [new_path, legacy_path] - for candidate_smi in smis: - if os.path.exists(candidate_smi): - smi = f'"{candidate_smi}"' - break - return smi - - -def get_platform(): - if sys.platform.startswith("linux"): - return "linux" - elif sys.platform.startswith("win32"): - return "win32" - elif sys.platform.startswith("cygwin"): - return "cygwin" - elif sys.platform.startswith("darwin"): - return "darwin" - else: - return sys.platform - - -def get_mac_version(run_lambda): - return run_and_parse_first_match(run_lambda, "sw_vers -productVersion", r"(.*)") - - -def get_windows_version(run_lambda): - system_root = os.environ.get("SYSTEMROOT", "C:\\Windows") - wmic_cmd = os.path.join(system_root, "System32", "Wbem", "wmic") - findstr_cmd = os.path.join(system_root, "System32", "findstr") - return run_and_read_all( - run_lambda, - f"{wmic_cmd} os get Caption | {findstr_cmd} /v Caption", - ) - - -def get_lsb_version(run_lambda): - return run_and_parse_first_match( - run_lambda, "lsb_release -a", r"Description:\t(.*)" - ) - - -def check_release_file(run_lambda): - return run_and_parse_first_match( - run_lambda, "cat /etc/*-release", r'PRETTY_NAME="(.*)"' - ) - - -def get_os(run_lambda): - from platform import machine - - platform = get_platform() - - if platform == "win32" or platform == "cygwin": - return get_windows_version(run_lambda) - - if platform == "darwin": - version = get_mac_version(run_lambda) - if version is None: - return None - return f"macOS {version} ({machine()})" - - if platform == "linux": - # Ubuntu/Debian based - desc = get_lsb_version(run_lambda) - if desc is not None: - return f"{desc} ({machine()})" - - # Try reading /etc/*-release - desc = check_release_file(run_lambda) - if desc is not None: - return f"{desc} ({machine()})" - - return f"{platform} ({machine()})" - - # Unknown platform - return platform - - -def get_pip_packages(run_lambda): - """Returns `pip list` output. Note: will also find conda-installed pytorch - and numpy packages.""" - - # People genly have `pip` as `pip` or `pip3` - def run_with_pip(pip): - if get_platform() == "win32": - system_root = os.environ.get("SYSTEMROOT", "C:\\Windows") - findstr_cmd = os.path.join(system_root, "System32", "findstr") - grep_cmd = rf'{findstr_cmd} /R "numpy torch"' - else: - grep_cmd = r'grep "torch\|numpy"' - return run_and_read_all(run_lambda, pip + " list --format=freeze | " + grep_cmd) - - # Try to figure out if the user is running pip or pip3. - out2 = run_with_pip("pip") - out3 = run_with_pip("pip3") - - num_pips = len([x for x in [out2, out3] if x is not None]) - if num_pips == 0: - return "pip", out2 - - if num_pips == 1: - if out2 is not None: - return "pip", out2 - return "pip3", out3 - - # num_pips is 2. Return pip3 by default b/c that most likely - # is the one associated with Python 3 - return "pip3", out3 - - -def get_env_info(): - run_lambda = run - pip_version, pip_list_output = get_pip_packages(run_lambda) - - if TORCH_AVAILABLE: - version_str = torch.__version__ - debug_mode_str = str(torch.version.debug) - cuda_available_str = str(torch.cuda.is_available()) - cuda_version_str = torch.version.cuda - if ( - not hasattr(torch.version, "hip") or torch.version.hip is None - ): # cuda version - hip_compiled_version = hip_runtime_version = miopen_runtime_version = "N/A" - else: # HIP version - cfg = torch._C._show_config().split("\n") - hip_runtime_version = [ - s.rsplit(None, 1)[-1] for s in cfg if "HIP Runtime" in s - ][0] - miopen_runtime_version = [ - s.rsplit(None, 1)[-1] for s in cfg if "MIOpen" in s - ][0] - cuda_version_str = "N/A" - hip_compiled_version = torch.version.hip - else: - version_str = debug_mode_str = cuda_available_str = cuda_version_str = "N/A" - hip_compiled_version = hip_runtime_version = miopen_runtime_version = "N/A" - - return SystemEnv( - torch_version=version_str, - is_debug_build=debug_mode_str, - python_version=f"{sys.version_info[0]}.{sys.version_info[1]} ({sys.maxsize.bit_length() + 1}-bit runtime)", - is_cuda_available=cuda_available_str, - cuda_compiled_version=cuda_version_str, - cuda_runtime_version=get_running_cuda_version(run_lambda), - nvidia_gpu_models=get_gpu_info(run_lambda), - nvidia_driver_version=get_nvidia_driver_version(run_lambda), - cudnn_version=get_cudnn_version(run_lambda), - hip_compiled_version=hip_compiled_version, - hip_runtime_version=hip_runtime_version, - miopen_runtime_version=miopen_runtime_version, - pip_version=pip_version, - pip_packages=pip_list_output, - conda_packages=get_conda_packages(run_lambda), - os=get_os(run_lambda), - gcc_version=get_gcc_version(run_lambda), - clang_version=get_clang_version(run_lambda), - cmake_version=get_cmake_version(run_lambda), - ) - - -env_info_fmt = """ -PyTorch version: {torch_version} -Is debug build: {is_debug_build} -CUDA used to build PyTorch: {cuda_compiled_version} -ROCM used to build PyTorch: {hip_compiled_version} - -OS: {os} -GCC version: {gcc_version} -Clang version: {clang_version} -CMake version: {cmake_version} - -Python version: {python_version} -Is CUDA available: {is_cuda_available} -CUDA runtime version: {cuda_runtime_version} -GPU models and configuration: {nvidia_gpu_models} -Nvidia driver version: {nvidia_driver_version} -cuDNN version: {cudnn_version} -HIP runtime version: {hip_runtime_version} -MIOpen runtime version: {miopen_runtime_version} - -Versions of relevant libraries: -{pip_packages} -{conda_packages} -""".strip() - - -def pretty_str(envinfo): - def replace_nones(dct, replacement="Could not collect"): - for key in dct.keys(): - if dct[key] is not None: - continue - dct[key] = replacement - return dct - - def replace_bools(dct, true="Yes", false="No"): - for key in dct.keys(): - if dct[key] is True: - dct[key] = true - elif dct[key] is False: - dct[key] = false - return dct - - def prepend(text, tag="[prepend]"): - lines = text.split("\n") - updated_lines = [tag + line for line in lines] - return "\n".join(updated_lines) - - def replace_if_empty(text, replacement="No relevant packages"): - if text is not None and len(text) == 0: - return replacement - return text - - def maybe_start_on_next_line(string): - # If `string` is multiline, prepend a \n to it. - if string is not None and len(string.split("\n")) > 1: - return f"\n{string}\n" - return string - - mutable_dict = envinfo._asdict() - - # If nvidia_gpu_models is multiline, start on the next line - mutable_dict["nvidia_gpu_models"] = maybe_start_on_next_line( - envinfo.nvidia_gpu_models - ) - - # If the machine doesn't have CUDA, report some fields as 'No CUDA' - dynamic_cuda_fields = [ - "cuda_runtime_version", - "nvidia_gpu_models", - "nvidia_driver_version", - ] - all_cuda_fields = dynamic_cuda_fields + ["cudnn_version"] - all_dynamic_cuda_fields_missing = all( - mutable_dict[field] is None for field in dynamic_cuda_fields - ) - if ( - TORCH_AVAILABLE - and not torch.cuda.is_available() - and all_dynamic_cuda_fields_missing - ): - for field in all_cuda_fields: - mutable_dict[field] = "No CUDA" - if envinfo.cuda_compiled_version is None: - mutable_dict["cuda_compiled_version"] = "None" - - # Replace True with Yes, False with No - mutable_dict = replace_bools(mutable_dict) - - # Replace all None objects with 'Could not collect' - mutable_dict = replace_nones(mutable_dict) - - # If either of these are '', replace with 'No relevant packages' - mutable_dict["pip_packages"] = replace_if_empty(mutable_dict["pip_packages"]) - mutable_dict["conda_packages"] = replace_if_empty(mutable_dict["conda_packages"]) - - # Tag conda and pip packages with a prefix - # If they were previously None, they'll show up as ie '[conda] Could not collect' - if mutable_dict["pip_packages"]: - mutable_dict["pip_packages"] = prepend( - mutable_dict["pip_packages"], f"[{envinfo.pip_version}] " - ) - if mutable_dict["conda_packages"]: - mutable_dict["conda_packages"] = prepend( - mutable_dict["conda_packages"], "[conda] " - ) - return env_info_fmt.format(**mutable_dict) - - -def get_pretty_env_info(): - return pretty_str(get_env_info()) - - -def main(): - print("Collecting environment information...") - output = get_pretty_env_info() - print(output) - - -if __name__ == "__main__": - main() diff --git a/src/scitex/resource/limit_ram.py b/src/scitex/resource/limit_ram.py deleted file mode 100755 index 757e1709d..000000000 --- a/src/scitex/resource/limit_ram.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Time-stamp: "2021-09-20 21:02:04 (ywatanabe)" - -import resource - -import scitex - - -def limit_ram(ram_factor): - soft, hard = resource.getrlimit(resource.RLIMIT_AS) - max_val = min(ram_factor * get_ram() * 1024, get_ram() * 1024) - resource.setrlimit(resource.RLIMIT_AS, (max_val, hard)) - print(f"\nFree RAM was limited to {scitex.gen.fmt_size(max_val)}") - - -def get_ram(): - with open("/proc/meminfo", "r") as mem: - free_memory = 0 - for i in mem: - sline = i.split() - if str(sline[0]) in ("MemFree:", "Buffers:", "Cached:"): - free_memory += int(sline[1]) - return free_memory - - -# Backward compatibility -limit_RAM = limit_ram # Deprecated: use limit_ram instead -get_RAM = get_ram # Deprecated: use get_ram instead - - -if __name__ == "__main__": - get_ram() - limit_ram(0.1) diff --git a/tests/scitex/resource/_utils/test__get_env_info.py b/tests/scitex/resource/_utils/test__get_env_info.py deleted file mode 100644 index c3f774a95..000000000 --- a/tests/scitex/resource/_utils/test__get_env_info.py +++ /dev/null @@ -1,1516 +0,0 @@ -#!/usr/bin/env python3 -# Timestamp: "2025-06-03 07:48:35 (ywatanabe)" -# File: ./tests/scitex/resource/_utils/test__get_env_info.py - -""" -Comprehensive tests for system environment information gathering functionality. -""" - -import pytest - -pytest.importorskip("zarr") -import os -import sys -from collections import namedtuple -from unittest.mock import MagicMock, Mock, patch - -# Import the module being tested -from scitex.resource._utils import ( - TORCH_AVAILABLE, - SystemEnv, - check_release_file, - env_info_fmt, - get_clang_version, - get_cmake_version, - get_conda_packages, - get_cudnn_version, - get_env_info, - get_gcc_version, - get_gpu_info, - get_lsb_version, - get_mac_version, - get_nvidia_driver_version, - get_nvidia_smi, - get_os, - get_pip_packages, - get_platform, - get_pretty_env_info, - get_running_cuda_version, - get_windows_version, - pretty_str, - run, - run_and_parse_first_match, - run_and_read_all, -) - - -class TestRunCommand: - """Test the run function for executing system commands.""" - - @patch("subprocess.Popen") - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="linux") - @patch("locale.getpreferredencoding", return_value="utf-8") - def test_run_success(self, mock_encoding, mock_platform, mock_popen): - """Test successful command execution.""" - mock_process = Mock() - mock_process.communicate.return_value = (b"output", b"") - mock_process.returncode = 0 - mock_popen.return_value = mock_process - - rc, output, err = run("echo test") - - assert rc == 0 - assert output == "output" - assert err == "" - mock_popen.assert_called_once() - - @pytest.mark.skipif(True, reason="OEM encoding not available in test environment") - def test_run_windows_encoding(self): - """Test Windows-specific encoding handling - skipped due to OEM encoding unavailability.""" - # This test would require Windows OEM encoding which is not available - # in the test environment. The actual Windows encoding logic is tested - # indirectly through integration tests on Windows systems. - pass - - @patch("subprocess.Popen") - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="linux") - @patch("locale.getpreferredencoding", return_value="utf-8") - def test_run_command_failure(self, mock_encoding, mock_platform, mock_popen): - """Test command execution failure.""" - mock_process = Mock() - mock_process.communicate.return_value = (b"", b"command not found") - mock_process.returncode = 127 - mock_popen.return_value = mock_process - - rc, output, err = run("nonexistent_command") - - assert rc == 127 - assert output == "" - assert err == "command not found" - - -class TestRunUtilities: - """Test utility functions for running commands.""" - - def test_run_and_read_all_success(self): - """Test successful command execution and output reading.""" - mock_run = Mock(return_value=(0, "success output", "")) - - result = run_and_read_all(mock_run, "test command") - - assert result == "success output" - mock_run.assert_called_once_with("test command") - - def test_run_and_read_all_failure(self): - """Test failed command execution.""" - mock_run = Mock(return_value=(1, "output", "error")) - - result = run_and_read_all(mock_run, "failing command") - - assert result is None - mock_run.assert_called_once_with("failing command") - - def test_run_and_parse_first_match_success(self): - """Test successful regex parsing.""" - mock_run = Mock(return_value=(0, "version 1.2.3 build", "")) - - result = run_and_parse_first_match(mock_run, "version cmd", r"version ([\d.]+)") - - assert result == "1.2.3" - mock_run.assert_called_once_with("version cmd") - - def test_run_and_parse_first_match_no_match(self): - """Test regex parsing with no match.""" - mock_run = Mock(return_value=(0, "no version here", "")) - - result = run_and_parse_first_match(mock_run, "cmd", r"version ([\d.]+)") - - assert result is None - - def test_run_and_parse_first_match_command_failure(self): - """Test regex parsing with command failure.""" - mock_run = Mock(return_value=(1, "", "error")) - - result = run_and_parse_first_match(mock_run, "cmd", r"pattern") - - assert result is None - - -class TestVersionParsing: - """Test version parsing functions.""" - - def test_get_gcc_version(self): - """Test GCC version parsing.""" - mock_run = Mock( - return_value=(0, "gcc (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0", "") - ) - - result = get_gcc_version(mock_run) - - assert result == "(Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0" - mock_run.assert_called_once_with("gcc --version") - - def test_get_clang_version(self): - """Test Clang version parsing.""" - mock_run = Mock(return_value=(0, "clang version 10.0.0-4ubuntu1", "")) - - result = get_clang_version(mock_run) - - assert result == "10.0.0-4ubuntu1" - mock_run.assert_called_once_with("clang --version") - - def test_get_cmake_version(self): - """Test CMake version parsing.""" - mock_run = Mock(return_value=(0, "cmake version 3.16.3", "")) - - result = get_cmake_version(mock_run) - - assert result == "version 3.16.3" - mock_run.assert_called_once_with("cmake --version") - - def test_get_running_cuda_version(self): - """Test running CUDA version parsing.""" - mock_run = Mock( - return_value=( - 0, - "nvcc: NVIDIA (R) Cuda compiler driver\nCopyright (c) 2005-2021 NVIDIA Corporation\nBuilt on release 11.4.48\nCuda compilation tools, release 11.4, V11.4.48", - "", - ) - ) - - result = get_running_cuda_version(mock_run) - - assert result == "11.4.48" - mock_run.assert_called_once_with("nvcc --version") - - def test_get_running_cuda_version_no_match(self): - """Test running CUDA version parsing with no match.""" - mock_run = Mock(return_value=(0, "no version info here", "")) - - result = get_running_cuda_version(mock_run) - - assert result is None - - -class TestPlatformDetection: - """Test platform detection functionality.""" - - @patch("sys.platform", "linux") - def test_get_platform_linux(self): - """Test Linux platform detection.""" - assert get_platform() == "linux" - - @patch("sys.platform", "win32") - def test_get_platform_windows(self): - """Test Windows platform detection.""" - assert get_platform() == "win32" - - @patch("sys.platform", "darwin") - def test_get_platform_macos(self): - """Test macOS platform detection.""" - assert get_platform() == "darwin" - - @patch("sys.platform", "cygwin") - def test_get_platform_cygwin(self): - """Test Cygwin platform detection.""" - assert get_platform() == "cygwin" - - @patch("sys.platform", "freebsd") - def test_get_platform_other(self): - """Test other platform detection.""" - assert get_platform() == "freebsd" - - -class TestNvidiaSMI: - """Test NVIDIA-smi path detection.""" - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="linux") - def test_get_nvidia_smi_linux(self, mock_platform): - """Test nvidia-smi path on Linux.""" - result = get_nvidia_smi() - assert result == "nvidia-smi" - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="win32") - @patch("os.path.exists") - @patch.dict( - os.environ, {"SYSTEMROOT": "C:\\Windows", "PROGRAMFILES": "C:\\Program Files"} - ) - def test_get_nvidia_smi_windows_new_path(self, mock_exists, mock_platform): - """Test nvidia-smi path on Windows (new location).""" - mock_exists.side_effect = lambda path: "System32" in path - - result = get_nvidia_smi() - - assert "System32" in result - assert "nvidia-smi" in result - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="win32") - @patch("os.path.exists") - @patch.dict( - os.environ, {"SYSTEMROOT": "C:\\Windows", "PROGRAMFILES": "C:\\Program Files"} - ) - def test_get_nvidia_smi_windows_legacy_path(self, mock_exists, mock_platform): - """Test nvidia-smi path on Windows (legacy location).""" - mock_exists.side_effect = lambda path: "NVIDIA Corporation" in path - - result = get_nvidia_smi() - - assert "NVIDIA Corporation" in result or result == "nvidia-smi" - - -class TestGPUInfo: - """Test GPU information gathering.""" - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="linux") - @patch( - "scitex.resource._utils._get_env_info.get_nvidia_smi", return_value="nvidia-smi" - ) - def test_get_gpu_info_nvidia(self, mock_smi, mock_platform): - """Test NVIDIA GPU information gathering.""" - mock_run = Mock( - return_value=(0, "GPU 0: NVIDIA GeForce RTX 3080 (UUID: GPU-12345)", "") - ) - - result = get_gpu_info(mock_run) - - assert "NVIDIA GeForce RTX 3080" in result - assert "UUID" not in result # Should be anonymized - mock_run.assert_called_once_with("nvidia-smi -L") - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="darwin") - @patch("scitex.resource._utils._get_env_info.TORCH_AVAILABLE", True) - def test_get_gpu_info_macos_torch(self, mock_platform): - """Test GPU info on macOS with PyTorch.""" - with patch("torch.cuda.is_available", return_value=True), patch( - "torch.cuda.get_device_name", return_value="Apple M1" - ): - result = get_gpu_info(Mock()) - assert result == "Apple M1" - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="linux") - @patch( - "scitex.resource._utils._get_env_info.get_nvidia_smi", return_value="nvidia-smi" - ) - def test_get_gpu_info_command_failure(self, mock_smi, mock_platform): - """Test GPU info gathering with command failure.""" - mock_run = Mock(return_value=(1, "", "nvidia-smi not found")) - - result = get_gpu_info(mock_run) - - assert result is None - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="linux") - @patch("scitex.resource._utils._get_env_info.TORCH_AVAILABLE", True) - def test_get_gpu_info_hip(self, mock_platform): - """Test GPU info gathering with HIP/ROCm.""" - with patch("torch.version.hip", "4.2.0"), patch( - "torch.cuda.is_available", return_value=True - ), patch("torch.cuda.get_device_name", return_value="AMD Radeon RX 6800 XT"): - result = get_gpu_info(Mock()) - assert result == "AMD Radeon RX 6800 XT" - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="linux") - @patch( - "scitex.resource._utils._get_env_info.get_nvidia_smi", return_value="nvidia-smi" - ) - def test_get_nvidia_driver_version_linux(self, mock_smi, mock_platform): - """Test NVIDIA driver version detection on Linux.""" - mock_run = Mock( - return_value=(0, "Driver Version: 470.57.02 CUDA Version: 11.4", "") - ) - - result = get_nvidia_driver_version(mock_run) - - assert result == "470.57.02" - mock_run.assert_called_once_with("nvidia-smi") - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="darwin") - def test_get_nvidia_driver_version_macos(self, mock_platform): - """Test NVIDIA driver version detection on macOS.""" - mock_run = Mock(return_value=(0, "com.nvidia.CUDA (418.105)", "")) - - result = get_nvidia_driver_version(mock_run) - - assert result == "418.105" - mock_run.assert_called_once_with("kextstat | grep -i cuda") - - -class TestCuDNNDetection: - """Test cuDNN detection functionality.""" - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="linux") - def test_get_cudnn_version_linux(self, mock_platform): - """Test cuDNN version detection on Linux.""" - mock_run = Mock( - return_value=( - 0, - "/usr/lib/x86_64-linux-gnu/libcudnn.so.8.2.1\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8", - "", - ) - ) - - with patch("os.path.realpath", side_effect=lambda x: x.strip()), patch( - "os.path.isfile", return_value=True - ): - result = get_cudnn_version(mock_run) - - assert isinstance(result, str) - assert "libcudnn.so.8" in result - mock_run.assert_called_once_with( - 'ldconfig -p | grep libcudnn | rev | cut -d" " -f1 | rev' - ) - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="win32") - @patch.dict( - os.environ, - { - "SYSTEMROOT": "C:\\Windows", - "CUDA_PATH": "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v11.4", - }, - ) - def test_get_cudnn_version_windows(self, mock_platform): - """Test cuDNN version detection on Windows.""" - mock_run = Mock( - return_value=( - 0, - "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v11.4\\bin\\cudnn64_8.dll", - "", - ) - ) - - with patch("os.path.realpath", side_effect=lambda x: x), patch( - "os.path.isfile", return_value=True - ): - result = get_cudnn_version(mock_run) - - assert "cudnn64_8.dll" in result - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="darwin") - def test_get_cudnn_version_macos(self, mock_platform): - """Test cuDNN version detection on macOS.""" - mock_run = Mock(return_value=(0, "/usr/local/cuda/lib/libcudnn.8.dylib", "")) - - with patch("os.path.realpath", side_effect=lambda x: x), patch( - "os.path.isfile", return_value=True - ): - result = get_cudnn_version(mock_run) - - assert "libcudnn.8.dylib" in result - mock_run.assert_called_once_with("ls /usr/local/cuda/lib/libcudnn*") - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="linux") - @patch.dict(os.environ, {"CUDNN_LIBRARY": "/custom/path/libcudnn.so"}) - def test_get_cudnn_version_custom_path(self, mock_platform): - """Test cuDNN version detection with custom CUDNN_LIBRARY path.""" - mock_run = Mock(return_value=(1, "", "")) # Command fails - - with patch("os.path.isfile", return_value=True), patch( - "os.path.realpath", return_value="/custom/path/libcudnn.so" - ): - result = get_cudnn_version(mock_run) - - assert result == "/custom/path/libcudnn.so" - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="linux") - def test_get_cudnn_version_not_found(self, mock_platform): - """Test cuDNN version when not found.""" - mock_run = Mock(return_value=(1, "", "not found")) - - result = get_cudnn_version(mock_run) - - assert result is None - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="linux") - def test_get_cudnn_version_multiple_files(self, mock_platform): - """Test cuDNN version with multiple library files.""" - mock_run = Mock( - return_value=( - 0, - "/usr/lib/libcudnn.so.8.2.1\n/usr/lib/libcudnn.so.8\n/usr/lib/libcudnn.so", - "", - ) - ) - - with patch("os.path.realpath", side_effect=lambda x: x.strip()), patch( - "os.path.isfile", return_value=True - ): - result = get_cudnn_version(mock_run) - - assert "Probably one of the following:" in result - assert "/usr/lib/libcudnn.so.8.2.1" in result - assert "/usr/lib/libcudnn.so.8" in result - assert "/usr/lib/libcudnn.so" in result - - -class TestPackageInfo: - """Test package information gathering.""" - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="linux") - @patch.dict(os.environ, {"CONDA_EXE": "conda"}) - def test_get_conda_packages_linux(self, mock_platform): - """Test conda package listing on Linux.""" - # Include comment at beginning of output which will be removed - mock_run = Mock( - return_value=( - 0, - "# This is a comment\ntorch=1.9.0\nnumpy=1.21.0\nother=1.0", - "", - ) - ) - - result = get_conda_packages(mock_run) - - assert "torch=1.9.0" in result - assert "numpy=1.21.0" in result - assert "# This is a comment" not in result # Comments at beginning are removed - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="win32") - @patch.dict(os.environ, {"SYSTEMROOT": "C:\\Windows", "CONDA_EXE": "conda"}) - def test_get_conda_packages_windows(self, mock_platform): - """Test conda package listing on Windows.""" - mock_run = Mock(return_value=(0, "torch=1.9.0\nnumpy=1.21.0", "")) - - result = get_conda_packages(mock_run) - - assert "torch=1.9.0" in result - assert "numpy=1.21.0" in result - - def test_get_pip_packages_pip_only(self): - """Test pip package gathering with pip only.""" - mock_run_pip = Mock(return_value=(0, "torch==1.9.0\nnumpy==1.21.0", "")) - mock_run_pip3 = Mock(return_value=(1, "", "command not found")) - - def mock_run_side_effect(cmd): - if "pip list" in cmd and "pip3" not in cmd: - return mock_run_pip(cmd) - else: - return mock_run_pip3(cmd) - - mock_run = Mock(side_effect=mock_run_side_effect) - - pip_version, packages = get_pip_packages(mock_run) - - assert pip_version == "pip" - assert "torch==1.9.0" in packages - - def test_get_pip_packages_pip3_only(self): - """Test pip package gathering with pip3 only.""" - mock_run_pip = Mock(return_value=(1, "", "command not found")) - mock_run_pip3 = Mock(return_value=(0, "torch==1.9.0\nnumpy==1.21.0", "")) - - def mock_run_side_effect(cmd): - if "pip3 list" in cmd: - return mock_run_pip3(cmd) - else: - return mock_run_pip(cmd) - - mock_run = Mock(side_effect=mock_run_side_effect) - - pip_version, packages = get_pip_packages(mock_run) - - assert pip_version == "pip3" - assert "torch==1.9.0" in packages - - def test_get_pip_packages_both_available(self): - """Test pip package gathering with both pip and pip3.""" - mock_run_pip = Mock(return_value=(0, "torch==1.9.0", "")) - mock_run_pip3 = Mock(return_value=(0, "torch==1.9.1\nnumpy==1.21.0", "")) - - def mock_run_side_effect(cmd): - if "pip3 list" in cmd: - return mock_run_pip3(cmd) - else: - return mock_run_pip(cmd) - - mock_run = Mock(side_effect=mock_run_side_effect) - - pip_version, packages = get_pip_packages(mock_run) - - assert pip_version == "pip3" # Should prefer pip3 - assert "torch==1.9.1" in packages - - def test_get_pip_packages_none_available(self): - """Test pip package gathering when neither pip nor pip3 is available.""" - mock_run = Mock(return_value=(1, "", "command not found")) - - pip_version, packages = get_pip_packages(mock_run) - - assert pip_version == "pip" - assert packages is None - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="win32") - @patch.dict(os.environ, {"SYSTEMROOT": "C:\\Windows"}) - def test_get_pip_packages_windows(self, mock_platform): - """Test pip package gathering on Windows.""" - mock_run = Mock(return_value=(0, "torch==1.9.0\nnumpy==1.21.0", "")) - - pip_version, packages = get_pip_packages(mock_run) - - # Should use findstr on Windows - call_arg = mock_run.call_args[0][0] - assert "findstr" in call_arg - assert packages == "torch==1.9.0\nnumpy==1.21.0" - - -class TestOSDetection: - """Test operating system detection.""" - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="linux") - @patch("platform.machine", return_value="x86_64") - def test_get_os_linux_lsb(self, mock_machine, mock_platform): - """Test Linux OS detection with lsb_release.""" - mock_run = Mock() - mock_run.side_effect = [ - (0, "Description:\tUbuntu 20.04.3 LTS", ""), # lsb_release - ] - - result = get_os(mock_run) - - assert "Ubuntu 20.04.3 LTS (x86_64)" == result - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="linux") - @patch("platform.machine", return_value="x86_64") - def test_get_os_linux_release_file(self, mock_machine, mock_platform): - """Test Linux OS detection with release file.""" - mock_run = Mock() - mock_run.side_effect = [ - (1, "", "command not found"), # lsb_release fails - (0, 'PRETTY_NAME="Ubuntu 20.04.3 LTS"', ""), # /etc/*-release - ] - - result = get_os(mock_run) - - assert "Ubuntu 20.04.3 LTS (x86_64)" == result - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="darwin") - @patch("platform.machine", return_value="arm64") - def test_get_os_macos(self, mock_machine, mock_platform): - """Test macOS detection.""" - mock_run = Mock(return_value=(0, "12.0.1", "")) - - result = get_os(mock_run) - - assert "macOS 12.0.1 (arm64)" == result - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="win32") - @patch.dict(os.environ, {"SYSTEMROOT": "C:\\Windows"}) - def test_get_os_windows(self, mock_platform): - """Test Windows detection.""" - mock_run = Mock(return_value=(0, "Microsoft Windows 11 Home", "")) - - result = get_os(mock_run) - - assert result == "Microsoft Windows 11 Home" - - def test_get_mac_version(self): - """Test macOS version detection.""" - mock_run = Mock(return_value=(0, "12.0.1", "")) - - result = get_mac_version(mock_run) - - assert result == "12.0.1" - mock_run.assert_called_once_with("sw_vers -productVersion") - - def test_get_windows_version(self): - """Test Windows version detection.""" - with patch.dict(os.environ, {"SYSTEMROOT": "C:\\Windows"}): - mock_run = Mock(return_value=(0, "Microsoft Windows 11 Home", "")) - - result = get_windows_version(mock_run) - - assert result == "Microsoft Windows 11 Home" - # Check that wmic and findstr commands were used - assert "wmic" in mock_run.call_args[0][0] - assert "findstr" in mock_run.call_args[0][0] - - def test_get_lsb_version(self): - """Test LSB version detection.""" - mock_run = Mock( - return_value=( - 0, - "Distributor ID:\tUbuntu\nDescription:\tUbuntu 20.04.3 LTS\nRelease:\t20.04", - "", - ) - ) - - result = get_lsb_version(mock_run) - - assert result == "Ubuntu 20.04.3 LTS" - mock_run.assert_called_once_with("lsb_release -a") - - def test_check_release_file(self): - """Test release file checking.""" - mock_run = Mock( - return_value=( - 0, - 'NAME="Ubuntu"\nVERSION="20.04.3 LTS (Focal Fossa)"\nPRETTY_NAME="Ubuntu 20.04.3 LTS"', - "", - ) - ) - - result = check_release_file(mock_run) - - assert result == "Ubuntu 20.04.3 LTS" - mock_run.assert_called_once_with("cat /etc/*-release") - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="linux") - @patch("platform.machine", return_value="x86_64") - def test_get_os_linux_fallback(self, mock_machine, mock_platform): - """Test Linux OS detection fallback when all methods fail.""" - mock_run = Mock() - mock_run.side_effect = [ - (1, "", "command not found"), # lsb_release fails - (1, "", "no such file"), # /etc/*-release fails - ] - - result = get_os(mock_run) - - assert result == "linux (x86_64)" - - @patch("scitex.resource._utils._get_env_info.get_platform", return_value="freebsd") - def test_get_os_unknown_platform(self, mock_platform): - """Test OS detection for unknown platforms.""" - mock_run = Mock() - - result = get_os(mock_run) - - assert result == "freebsd" - - -class TestEnvironmentInfo: - """Test main environment info gathering function.""" - - @patch("scitex.resource._utils._get_env_info.TORCH_AVAILABLE", True) - @patch("scitex.resource._utils._get_env_info.get_pip_packages") - @patch("scitex.resource._utils._get_env_info.get_conda_packages") - @patch("scitex.resource._utils._get_env_info.get_os") - def test_get_env_info_with_torch(self, mock_os, mock_conda, mock_pip): - """Test environment info gathering with PyTorch available.""" - mock_pip.return_value = ("pip3", "torch==1.9.0") - mock_conda.return_value = "torch=1.9.0" - mock_os.return_value = "Ubuntu 20.04" - - with patch("torch.__version__", "1.9.0"), patch( - "torch.version.debug", False - ), patch("torch.cuda.is_available", return_value=True), patch( - "torch.version.cuda", "11.1" - ), patch( - "torch.version.hip", None - ): - result = get_env_info() - - assert isinstance(result, SystemEnv) - assert result.torch_version == "1.9.0" - assert result.is_debug_build == "False" - assert result.is_cuda_available == "True" - assert result.cuda_compiled_version == "11.1" - assert result.os == "Ubuntu 20.04" - - @patch("scitex.resource._utils._get_env_info.TORCH_AVAILABLE", False) - @patch("scitex.resource._utils._get_env_info.get_pip_packages") - @patch("scitex.resource._utils._get_env_info.get_conda_packages") - @patch("scitex.resource._utils._get_env_info.get_os") - def test_get_env_info_without_torch(self, mock_os, mock_conda, mock_pip): - """Test environment info gathering without PyTorch.""" - mock_pip.return_value = ("pip3", "numpy==1.21.0") - mock_conda.return_value = "numpy=1.21.0" - mock_os.return_value = "Ubuntu 20.04" - - result = get_env_info() - - assert isinstance(result, SystemEnv) - assert result.torch_version == "N/A" - assert result.is_debug_build == "N/A" - assert result.is_cuda_available == "N/A" - assert result.cuda_compiled_version == "N/A" - - @patch("scitex.resource._utils._get_env_info.TORCH_AVAILABLE", True) - @patch("scitex.resource._utils._get_env_info.get_pip_packages") - @patch("scitex.resource._utils._get_env_info.get_conda_packages") - @patch("scitex.resource._utils._get_env_info.get_os") - @patch("scitex.resource._utils._get_env_info.get_running_cuda_version") - @patch("scitex.resource._utils._get_env_info.get_gpu_info") - @patch("scitex.resource._utils._get_env_info.get_nvidia_driver_version") - @patch("scitex.resource._utils._get_env_info.get_cudnn_version") - @patch("scitex.resource._utils._get_env_info.get_gcc_version") - @patch("scitex.resource._utils._get_env_info.get_clang_version") - @patch("scitex.resource._utils._get_env_info.get_cmake_version") - def test_get_env_info_with_hip( - self, - mock_cmake, - mock_clang, - mock_gcc, - mock_cudnn, - mock_driver, - mock_gpu, - mock_cuda_ver, - mock_os, - mock_conda, - mock_pip, - ): - """Test environment info gathering with HIP/ROCm support.""" - mock_pip.return_value = ("pip3", "torch==1.9.0+rocm4.2") - mock_conda.return_value = "" - mock_os.return_value = "Ubuntu 20.04" - mock_cuda_ver.return_value = None - mock_gpu.return_value = "AMD Radeon RX 6800 XT" - mock_driver.return_value = None - mock_cudnn.return_value = None - mock_gcc.return_value = "9.4.0" - mock_clang.return_value = None - mock_cmake.return_value = "3.16.3" - - with patch("torch.__version__", "1.9.0+rocm4.2"), patch( - "torch.version.debug", False - ), patch("torch.cuda.is_available", return_value=True), patch( - "torch.version.cuda", None - ), patch( - "torch.version.hip", "4.2.0" - ), patch( - "torch._C._show_config", return_value="HIP Runtime: 4.2.0\nMIOpen: 2.14.0" - ): - result = get_env_info() - - assert isinstance(result, SystemEnv) - assert result.torch_version == "1.9.0+rocm4.2" - assert result.cuda_compiled_version == "N/A" - assert result.hip_compiled_version == "4.2.0" - assert result.hip_runtime_version == "4.2.0" - assert result.miopen_runtime_version == "2.14.0" - - -class TestPrettyFormatting: - """Test pretty string formatting functionality.""" - - def test_pretty_str_basic_formatting(self): - """Test basic pretty string formatting.""" - env_info = SystemEnv( - torch_version="1.9.0", - is_debug_build=False, - cuda_compiled_version="11.1", - gcc_version="9.4.0", - clang_version=None, - cmake_version="3.16.3", - os="Ubuntu 20.04", - python_version="3.8.10 (64-bit runtime)", - is_cuda_available=True, - cuda_runtime_version="11.2", - nvidia_driver_version="470.57.02", - nvidia_gpu_models="NVIDIA GeForce RTX 3080", - cudnn_version="8.2.1", - pip_version="pip3", - pip_packages="torch==1.9.0", - conda_packages="", - hip_compiled_version="N/A", - hip_runtime_version="N/A", - miopen_runtime_version="N/A", - ) - - result = pretty_str(env_info) - - assert "PyTorch version: 1.9.0" in result - assert "Is debug build: No" in result # False -> No - assert "Is CUDA available: Yes" in result # True -> Yes - assert "Could not collect" in result # None -> Could not collect - assert "[pip3] torch==1.9.0" in result # Prefixed packages - assert "No relevant packages" in result # Empty conda packages - - def test_pretty_str_multiline_gpu_models(self): - """Test formatting with multiline GPU models.""" - env_info = SystemEnv( - torch_version="1.9.0", - is_debug_build=False, - cuda_compiled_version="11.1", - gcc_version="9.4.0", - clang_version=None, - cmake_version="3.16.3", - os="Ubuntu 20.04", - python_version="3.8.10 (64-bit runtime)", - is_cuda_available=True, - cuda_runtime_version="11.2", - nvidia_driver_version="470.57.02", - nvidia_gpu_models="GPU 0: NVIDIA GeForce RTX 3080\nGPU 1: NVIDIA GeForce RTX 3090", - cudnn_version="8.2.1", - pip_version="pip3", - pip_packages="torch==1.9.0", - conda_packages="torch=1.9.0", - hip_compiled_version="N/A", - hip_runtime_version="N/A", - miopen_runtime_version="N/A", - ) - - result = pretty_str(env_info) - - assert "GPU 0: NVIDIA GeForce RTX 3080" in result - assert "GPU 1: NVIDIA GeForce RTX 3090" in result - - @patch("scitex.resource._utils._get_env_info.TORCH_AVAILABLE", True) - def test_pretty_str_no_cuda_available(self): - """Test formatting when CUDA is not available.""" - env_info = SystemEnv( - torch_version="1.9.0", - is_debug_build=False, - cuda_compiled_version=None, - gcc_version="9.4.0", - clang_version=None, - cmake_version="3.16.3", - os="Ubuntu 20.04", - python_version="3.8.10 (64-bit runtime)", - is_cuda_available=False, - cuda_runtime_version=None, - nvidia_driver_version=None, - nvidia_gpu_models=None, - cudnn_version=None, - pip_version="pip3", - pip_packages="torch==1.9.0", - conda_packages="torch=1.9.0", - hip_compiled_version="N/A", - hip_runtime_version="N/A", - miopen_runtime_version="N/A", - ) - - with patch("torch.cuda.is_available", return_value=False): - result = pretty_str(env_info) - - assert "CUDA used to build PyTorch: None" in result - assert "Is CUDA available: No" in result - assert "CUDA runtime version: No CUDA" in result - assert "GPU models and configuration: No CUDA" in result - assert "Nvidia driver version: No CUDA" in result - assert "cuDNN version: No CUDA" in result - - -class TestMainFunctions: - """Test main entry point functions.""" - - @patch("scitex.resource._utils._get_env_info.get_env_info") - def test_get_pretty_env_info(self, mock_get_env): - """Test pretty environment info function.""" - mock_env = SystemEnv( - torch_version="1.9.0", - is_debug_build=False, - cuda_compiled_version="11.1", - gcc_version="9.4.0", - clang_version=None, - cmake_version="3.16.3", - os="Ubuntu 20.04", - python_version="3.8.10 (64-bit runtime)", - is_cuda_available=True, - cuda_runtime_version="11.2", - nvidia_driver_version="470.57.02", - nvidia_gpu_models="NVIDIA GeForce RTX 3080", - cudnn_version="8.2.1", - pip_version="pip3", - pip_packages="torch==1.9.0", - conda_packages="torch=1.9.0", - hip_compiled_version="N/A", - hip_runtime_version="N/A", - miopen_runtime_version="N/A", - ) - mock_get_env.return_value = mock_env - - result = get_pretty_env_info() - - assert isinstance(result, str) - assert "PyTorch version: 1.9.0" in result - mock_get_env.assert_called_once() - - @patch("scitex.resource._utils._get_env_info.get_pretty_env_info") - @patch("builtins.print") - def test_main_function(self, mock_print, mock_get_pretty): - """Test main function execution.""" - from scitex.resource._utils import main - - mock_get_pretty.return_value = "PyTorch version: 1.9.0\nOS: Ubuntu 20.04" - - main() - - # Should print the collection message and environment info - assert mock_print.call_count == 2 - mock_print.assert_any_call("Collecting environment information...") - mock_print.assert_any_call("PyTorch version: 1.9.0\nOS: Ubuntu 20.04") - mock_get_pretty.assert_called_once() - - -class TestSystemEnvNamedTuple: - """Test SystemEnv namedtuple functionality.""" - - def test_system_env_creation(self): - """Test SystemEnv namedtuple creation.""" - env = SystemEnv( - torch_version="1.9.0", - is_debug_build=False, - cuda_compiled_version="11.1", - gcc_version="9.4.0", - clang_version="12.0.0", - cmake_version="3.16.3", - os="Ubuntu 20.04", - python_version="3.8.10 (64-bit runtime)", - is_cuda_available=True, - cuda_runtime_version="11.2", - nvidia_driver_version="470.57.02", - nvidia_gpu_models="NVIDIA GeForce RTX 3080", - cudnn_version="8.2.1", - pip_version="pip3", - pip_packages="torch==1.9.0", - conda_packages="torch=1.9.0", - hip_compiled_version="N/A", - hip_runtime_version="N/A", - miopen_runtime_version="N/A", - ) - - assert env.torch_version == "1.9.0" - assert env.is_debug_build is False - assert env.cuda_compiled_version == "11.1" - assert env.os == "Ubuntu 20.04" - assert len(env) == 19 # Check all fields are present - - def test_system_env_asdict(self): - """Test SystemEnv to dict conversion.""" - env = SystemEnv( - torch_version="1.9.0", - is_debug_build=False, - cuda_compiled_version="11.1", - gcc_version="9.4.0", - clang_version="12.0.0", - cmake_version="3.16.3", - os="Ubuntu 20.04", - python_version="3.8.10 (64-bit runtime)", - is_cuda_available=True, - cuda_runtime_version="11.2", - nvidia_driver_version="470.57.02", - nvidia_gpu_models="NVIDIA GeForce RTX 3080", - cudnn_version="8.2.1", - pip_version="pip3", - pip_packages="torch==1.9.0", - conda_packages="torch=1.9.0", - hip_compiled_version="N/A", - hip_runtime_version="N/A", - miopen_runtime_version="N/A", - ) - - env_dict = env._asdict() - - assert isinstance(env_dict, dict) - assert env_dict["torch_version"] == "1.9.0" - assert env_dict["os"] == "Ubuntu 20.04" - assert len(env_dict) == 19 - - -class TestConstantsAndFormats: - """Test module constants and format strings.""" - - def test_env_info_fmt_contains_required_fields(self): - """Test that format string contains all required fields.""" - required_fields = [ - "torch_version", - "is_debug_build", - "cuda_compiled_version", - "hip_compiled_version", - "os", - "gcc_version", - "clang_version", - "cmake_version", - "python_version", - "is_cuda_available", - "cuda_runtime_version", - "nvidia_gpu_models", - "nvidia_driver_version", - "cudnn_version", - "hip_runtime_version", - "miopen_runtime_version", - "pip_packages", - "conda_packages", - ] - - for field in required_fields: - assert "{" + field + "}" in env_info_fmt - - def test_torch_available_constant(self): - """Test TORCH_AVAILABLE constant is boolean.""" - assert isinstance(TORCH_AVAILABLE, bool) - - -if __name__ == "__main__": - import os - - import pytest - - pytest.main([os.path.abspath(__file__)]) - -# -------------------------------------------------------------------------------- -# Start of Source Code from: /home/ywatanabe/proj/scitex-code/src/scitex/resource/_utils/_get_env_info.py -# -------------------------------------------------------------------------------- -# #!/usr/bin/env python3 -# -# # This script outputs relevant system environment info -# # Run it with `python collect_env.py`. -# import locale -# import os -# import re -# import subprocess -# import sys -# from collections import namedtuple -# -# try: -# import torch -# -# TORCH_AVAILABLE = True -# except (ImportError, NameError, AttributeError, OSError): -# TORCH_AVAILABLE = False -# -# # System Environment Information -# SystemEnv = namedtuple( -# "SystemEnv", -# [ -# "torch_version", -# "is_debug_build", -# "cuda_compiled_version", -# "gcc_version", -# "clang_version", -# "cmake_version", -# "os", -# "python_version", -# "is_cuda_available", -# "cuda_runtime_version", -# "nvidia_driver_version", -# "nvidia_gpu_models", -# "cudnn_version", -# "pip_version", # 'pip' or 'pip3' -# "pip_packages", -# "conda_packages", -# "hip_compiled_version", -# "hip_runtime_version", -# "miopen_runtime_version", -# ], -# ) -# -# -# def run(command): -# """Returns (return-code, stdout, stderr)""" -# p = subprocess.Popen( -# command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True -# ) -# raw_output, raw_err = p.communicate() -# rc = p.returncode -# if get_platform() == "win32": -# enc = "oem" -# else: -# enc = locale.getpreferredencoding() -# output = raw_output.decode(enc) -# err = raw_err.decode(enc) -# return rc, output.strip(), err.strip() -# -# -# def run_and_read_all(run_lambda, command): -# """Runs command using run_lambda; reads and returns entire output if rc is 0""" -# rc, out, _ = run_lambda(command) -# if rc != 0: -# return None -# return out -# -# -# def run_and_parse_first_match(run_lambda, command, regex): -# """Runs command using run_lambda, returns the first regex match if it exists""" -# rc, out, _ = run_lambda(command) -# if rc != 0: -# return None -# match = re.search(regex, out) -# if match is None: -# return None -# return match.group(1) -# -# -# def get_conda_packages(run_lambda): -# if get_platform() == "win32": -# system_root = os.environ.get("SYSTEMROOT", "C:\\Windows") -# findstr_cmd = os.path.join(system_root, "System32", "findstr") -# grep_cmd = rf'{findstr_cmd} /R "torch numpy cudatoolkit soumith mkl magma"' -# else: -# grep_cmd = r'grep "torch\|numpy\|cudatoolkit\|soumith\|mkl\|magma"' -# conda = os.environ.get("CONDA_EXE", "conda") -# out = run_and_read_all(run_lambda, conda + " list | " + grep_cmd) -# if out is None: -# return out -# # Comment starting at beginning of line -# comment_regex = re.compile(r"^#.*\n") -# return re.sub(comment_regex, "", out) -# -# -# def get_gcc_version(run_lambda): -# return run_and_parse_first_match(run_lambda, "gcc --version", r"gcc (.*)") -# -# -# def get_clang_version(run_lambda): -# return run_and_parse_first_match( -# run_lambda, "clang --version", r"clang version (.*)" -# ) -# -# -# def get_cmake_version(run_lambda): -# return run_and_parse_first_match(run_lambda, "cmake --version", r"cmake (.*)") -# -# -# def get_nvidia_driver_version(run_lambda): -# if get_platform() == "darwin": -# cmd = "kextstat | grep -i cuda" -# return run_and_parse_first_match( -# run_lambda, cmd, r"com[.]nvidia[.]CUDA [(](.*?)[)]" -# ) -# smi = get_nvidia_smi() -# return run_and_parse_first_match(run_lambda, smi, r"Driver Version: (.*?) ") -# -# -# def get_gpu_info(run_lambda): -# if get_platform() == "darwin" or ( -# TORCH_AVAILABLE -# and hasattr(torch.version, "hip") -# and torch.version.hip is not None -# ): -# if TORCH_AVAILABLE and torch.cuda.is_available(): -# return torch.cuda.get_device_name(None) -# return None -# smi = get_nvidia_smi() -# uuid_regex = re.compile(r" \(UUID: .+?\)") -# rc, out, _ = run_lambda(smi + " -L") -# if rc != 0: -# return None -# # Anonymize GPUs by removing their UUID -# return re.sub(uuid_regex, "", out) -# -# -# def get_running_cuda_version(run_lambda): -# return run_and_parse_first_match(run_lambda, "nvcc --version", r"release .+ V(.*)") -# -# -# def get_cudnn_version(run_lambda): -# """This will return a list of libcudnn.so; it's hard to tell which one is being used""" -# if get_platform() == "win32": -# system_root = os.environ.get("SYSTEMROOT", "C:\\Windows") -# cuda_path = os.environ.get("CUDA_PATH", "%CUDA_PATH%") -# where_cmd = os.path.join(system_root, "System32", "where") -# cudnn_cmd = f'{where_cmd} /R "{cuda_path}\\bin" cudnn*.dll' -# elif get_platform() == "darwin": -# # CUDA libraries and drivers can be found in /usr/local/cuda/. See -# # https://docs.nvidia.com/cuda/cuda-installation-guide-mac-os-x/index.html#install -# # https://docs.nvidia.com/deeplearning/sdk/cudnn-install/index.html#installmac -# # Use CUDNN_LIBRARY when cudnn library is installed elsewhere. -# cudnn_cmd = "ls /usr/local/cuda/lib/libcudnn*" -# else: -# cudnn_cmd = 'ldconfig -p | grep libcudnn | rev | cut -d" " -f1 | rev' -# rc, out, _ = run_lambda(cudnn_cmd) -# # find will return 1 if there are permission errors or if not found -# if len(out) == 0 or (rc != 1 and rc != 0): -# l = os.environ.get("CUDNN_LIBRARY") -# if l is not None and os.path.isfile(l): -# return os.path.realpath(l) -# return None -# files_set = set() -# for fn in out.split("\n"): -# fn = os.path.realpath(fn) # eliminate symbolic links -# if os.path.isfile(fn): -# files_set.add(fn) -# if not files_set: -# return None -# # Alphabetize the result because the order is non-deterministic otherwise -# files = sorted(files_set) -# if len(files) == 1: -# return files[0] -# result = "\n".join(files) -# return f"Probably one of the following:\n{result}" -# -# -# def get_nvidia_smi(): -# # Note: nvidia-smi is currently available only on Windows and Linux -# smi = "nvidia-smi" -# if get_platform() == "win32": -# system_root = os.environ.get("SYSTEMROOT", "C:\\Windows") -# program_files_root = os.environ.get("PROGRAMFILES", "C:\\Program Files") -# legacy_path = os.path.join( -# program_files_root, "NVIDIA Corporation", "NVSMI", smi -# ) -# new_path = os.path.join(system_root, "System32", smi) -# smis = [new_path, legacy_path] -# for candidate_smi in smis: -# if os.path.exists(candidate_smi): -# smi = f'"{candidate_smi}"' -# break -# return smi -# -# -# def get_platform(): -# if sys.platform.startswith("linux"): -# return "linux" -# elif sys.platform.startswith("win32"): -# return "win32" -# elif sys.platform.startswith("cygwin"): -# return "cygwin" -# elif sys.platform.startswith("darwin"): -# return "darwin" -# else: -# return sys.platform -# -# -# def get_mac_version(run_lambda): -# return run_and_parse_first_match(run_lambda, "sw_vers -productVersion", r"(.*)") -# -# -# def get_windows_version(run_lambda): -# system_root = os.environ.get("SYSTEMROOT", "C:\\Windows") -# wmic_cmd = os.path.join(system_root, "System32", "Wbem", "wmic") -# findstr_cmd = os.path.join(system_root, "System32", "findstr") -# return run_and_read_all( -# run_lambda, -# f"{wmic_cmd} os get Caption | {findstr_cmd} /v Caption", -# ) -# -# -# def get_lsb_version(run_lambda): -# return run_and_parse_first_match( -# run_lambda, "lsb_release -a", r"Description:\t(.*)" -# ) -# -# -# def check_release_file(run_lambda): -# return run_and_parse_first_match( -# run_lambda, "cat /etc/*-release", r'PRETTY_NAME="(.*)"' -# ) -# -# -# def get_os(run_lambda): -# from platform import machine -# -# platform = get_platform() -# -# if platform == "win32" or platform == "cygwin": -# return get_windows_version(run_lambda) -# -# if platform == "darwin": -# version = get_mac_version(run_lambda) -# if version is None: -# return None -# return f"macOS {version} ({machine()})" -# -# if platform == "linux": -# # Ubuntu/Debian based -# desc = get_lsb_version(run_lambda) -# if desc is not None: -# return f"{desc} ({machine()})" -# -# # Try reading /etc/*-release -# desc = check_release_file(run_lambda) -# if desc is not None: -# return f"{desc} ({machine()})" -# -# return f"{platform} ({machine()})" -# -# # Unknown platform -# return platform -# -# -# def get_pip_packages(run_lambda): -# """Returns `pip list` output. Note: will also find conda-installed pytorch -# and numpy packages.""" -# -# # People genly have `pip` as `pip` or `pip3` -# def run_with_pip(pip): -# if get_platform() == "win32": -# system_root = os.environ.get("SYSTEMROOT", "C:\\Windows") -# findstr_cmd = os.path.join(system_root, "System32", "findstr") -# grep_cmd = rf'{findstr_cmd} /R "numpy torch"' -# else: -# grep_cmd = r'grep "torch\|numpy"' -# return run_and_read_all(run_lambda, pip + " list --format=freeze | " + grep_cmd) -# -# # Try to figure out if the user is running pip or pip3. -# out2 = run_with_pip("pip") -# out3 = run_with_pip("pip3") -# -# num_pips = len([x for x in [out2, out3] if x is not None]) -# if num_pips == 0: -# return "pip", out2 -# -# if num_pips == 1: -# if out2 is not None: -# return "pip", out2 -# return "pip3", out3 -# -# # num_pips is 2. Return pip3 by default b/c that most likely -# # is the one associated with Python 3 -# return "pip3", out3 -# -# -# def get_env_info(): -# run_lambda = run -# pip_version, pip_list_output = get_pip_packages(run_lambda) -# -# if TORCH_AVAILABLE: -# version_str = torch.__version__ -# debug_mode_str = str(torch.version.debug) -# cuda_available_str = str(torch.cuda.is_available()) -# cuda_version_str = torch.version.cuda -# if ( -# not hasattr(torch.version, "hip") or torch.version.hip is None -# ): # cuda version -# hip_compiled_version = hip_runtime_version = miopen_runtime_version = "N/A" -# else: # HIP version -# cfg = torch._C._show_config().split("\n") -# hip_runtime_version = [ -# s.rsplit(None, 1)[-1] for s in cfg if "HIP Runtime" in s -# ][0] -# miopen_runtime_version = [ -# s.rsplit(None, 1)[-1] for s in cfg if "MIOpen" in s -# ][0] -# cuda_version_str = "N/A" -# hip_compiled_version = torch.version.hip -# else: -# version_str = debug_mode_str = cuda_available_str = cuda_version_str = "N/A" -# hip_compiled_version = hip_runtime_version = miopen_runtime_version = "N/A" -# -# return SystemEnv( -# torch_version=version_str, -# is_debug_build=debug_mode_str, -# python_version=f"{sys.version_info[0]}.{sys.version_info[1]} ({sys.maxsize.bit_length() + 1}-bit runtime)", -# is_cuda_available=cuda_available_str, -# cuda_compiled_version=cuda_version_str, -# cuda_runtime_version=get_running_cuda_version(run_lambda), -# nvidia_gpu_models=get_gpu_info(run_lambda), -# nvidia_driver_version=get_nvidia_driver_version(run_lambda), -# cudnn_version=get_cudnn_version(run_lambda), -# hip_compiled_version=hip_compiled_version, -# hip_runtime_version=hip_runtime_version, -# miopen_runtime_version=miopen_runtime_version, -# pip_version=pip_version, -# pip_packages=pip_list_output, -# conda_packages=get_conda_packages(run_lambda), -# os=get_os(run_lambda), -# gcc_version=get_gcc_version(run_lambda), -# clang_version=get_clang_version(run_lambda), -# cmake_version=get_cmake_version(run_lambda), -# ) -# -# -# env_info_fmt = """ -# PyTorch version: {torch_version} -# Is debug build: {is_debug_build} -# CUDA used to build PyTorch: {cuda_compiled_version} -# ROCM used to build PyTorch: {hip_compiled_version} -# -# OS: {os} -# GCC version: {gcc_version} -# Clang version: {clang_version} -# CMake version: {cmake_version} -# -# Python version: {python_version} -# Is CUDA available: {is_cuda_available} -# CUDA runtime version: {cuda_runtime_version} -# GPU models and configuration: {nvidia_gpu_models} -# Nvidia driver version: {nvidia_driver_version} -# cuDNN version: {cudnn_version} -# HIP runtime version: {hip_runtime_version} -# MIOpen runtime version: {miopen_runtime_version} -# -# Versions of relevant libraries: -# {pip_packages} -# {conda_packages} -# """.strip() -# -# -# def pretty_str(envinfo): -# def replace_nones(dct, replacement="Could not collect"): -# for key in dct.keys(): -# if dct[key] is not None: -# continue -# dct[key] = replacement -# return dct -# -# def replace_bools(dct, true="Yes", false="No"): -# for key in dct.keys(): -# if dct[key] is True: -# dct[key] = true -# elif dct[key] is False: -# dct[key] = false -# return dct -# -# def prepend(text, tag="[prepend]"): -# lines = text.split("\n") -# updated_lines = [tag + line for line in lines] -# return "\n".join(updated_lines) -# -# def replace_if_empty(text, replacement="No relevant packages"): -# if text is not None and len(text) == 0: -# return replacement -# return text -# -# def maybe_start_on_next_line(string): -# # If `string` is multiline, prepend a \n to it. -# if string is not None and len(string.split("\n")) > 1: -# return f"\n{string}\n" -# return string -# -# mutable_dict = envinfo._asdict() -# -# # If nvidia_gpu_models is multiline, start on the next line -# mutable_dict["nvidia_gpu_models"] = maybe_start_on_next_line( -# envinfo.nvidia_gpu_models -# ) -# -# # If the machine doesn't have CUDA, report some fields as 'No CUDA' -# dynamic_cuda_fields = [ -# "cuda_runtime_version", -# "nvidia_gpu_models", -# "nvidia_driver_version", -# ] -# all_cuda_fields = dynamic_cuda_fields + ["cudnn_version"] -# all_dynamic_cuda_fields_missing = all( -# mutable_dict[field] is None for field in dynamic_cuda_fields -# ) -# if ( -# TORCH_AVAILABLE -# and not torch.cuda.is_available() -# and all_dynamic_cuda_fields_missing -# ): -# for field in all_cuda_fields: -# mutable_dict[field] = "No CUDA" -# if envinfo.cuda_compiled_version is None: -# mutable_dict["cuda_compiled_version"] = "None" -# -# # Replace True with Yes, False with No -# mutable_dict = replace_bools(mutable_dict) -# -# # Replace all None objects with 'Could not collect' -# mutable_dict = replace_nones(mutable_dict) -# -# # If either of these are '', replace with 'No relevant packages' -# mutable_dict["pip_packages"] = replace_if_empty(mutable_dict["pip_packages"]) -# mutable_dict["conda_packages"] = replace_if_empty(mutable_dict["conda_packages"]) -# -# # Tag conda and pip packages with a prefix -# # If they were previously None, they'll show up as ie '[conda] Could not collect' -# if mutable_dict["pip_packages"]: -# mutable_dict["pip_packages"] = prepend( -# mutable_dict["pip_packages"], f"[{envinfo.pip_version}] " -# ) -# if mutable_dict["conda_packages"]: -# mutable_dict["conda_packages"] = prepend( -# mutable_dict["conda_packages"], "[conda] " -# ) -# return env_info_fmt.format(**mutable_dict) -# -# -# def get_pretty_env_info(): -# return pretty_str(get_env_info()) -# -# -# def main(): -# print("Collecting environment information...") -# output = get_pretty_env_info() -# print(output) -# -# -# if __name__ == "__main__": -# main() - -# -------------------------------------------------------------------------------- -# End of Source Code from: /home/ywatanabe/proj/scitex-code/src/scitex/resource/_utils/_get_env_info.py -# -------------------------------------------------------------------------------- diff --git a/tests/scitex/resource/test__get_processor_usages.py b/tests/scitex/resource/test__get_processor_usages.py deleted file mode 100644 index 348002673..000000000 --- a/tests/scitex/resource/test__get_processor_usages.py +++ /dev/null @@ -1,589 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Timestamp: "2025-06-02 15:00:00 (ywatanabe)" -# File: ./tests/scitex/resource/test__get_processor_usages.py - -"""Tests for processor usage monitoring functionality.""" - -import os -import subprocess -from datetime import datetime -from unittest.mock import MagicMock, Mock, patch - -import pandas as pd -import pytest - -pytest.importorskip("zarr") - -from scitex.resource import get_processor_usages -from scitex.resource._get_processor_usages import _get_cpu_usage, _get_gpu_usage - - -class TestGetProcessorUsages: - """Test suite for get_processor_usages function.""" - - @patch("scitex.resource._get_processor_usages._get_gpu_usage") - @patch("scitex.resource._get_processor_usages._get_cpu_usage") - def test_basic_functionality(self, mock_cpu, mock_gpu): - """Test basic processor usage retrieval.""" - mock_cpu.return_value = (25.3, 8.2) - mock_gpu.return_value = (65.0, 4.5) - - result = get_processor_usages() - - assert isinstance(result, pd.DataFrame) - assert len(result) == 1 - assert list(result.columns) == [ - "Timestamp", - "CPU [%]", - "RAM [GiB]", - "GPU [%]", - "VRAM [GiB]", - ] - assert result.iloc[0]["CPU [%]"] == 25.3 - assert result.iloc[0]["RAM [GiB]"] == 8.2 - assert result.iloc[0]["GPU [%]"] == 65.0 - assert result.iloc[0]["VRAM [GiB]"] == 4.5 - assert isinstance(result.iloc[0]["Timestamp"], datetime) - - @patch("scitex.resource._get_processor_usages._get_gpu_usage") - @patch("scitex.resource._get_processor_usages._get_cpu_usage") - def test_zero_usage(self, mock_cpu, mock_gpu): - """Test with zero resource usage.""" - mock_cpu.return_value = (0.0, 0.0) - mock_gpu.return_value = (0.0, 0.0) - - result = get_processor_usages() - - assert result.iloc[0]["CPU [%]"] == 0.0 - assert result.iloc[0]["RAM [GiB]"] == 0.0 - assert result.iloc[0]["GPU [%]"] == 0.0 - assert result.iloc[0]["VRAM [GiB]"] == 0.0 - - @patch("scitex.resource._get_processor_usages._get_gpu_usage") - @patch("scitex.resource._get_processor_usages._get_cpu_usage") - def test_high_usage(self, mock_cpu, mock_gpu): - """Test with high resource usage.""" - mock_cpu.return_value = (95.8, 31.7) - mock_gpu.return_value = (99.9, 23.8) - - result = get_processor_usages() - - assert result.iloc[0]["CPU [%]"] == 95.8 - assert result.iloc[0]["RAM [GiB]"] == 31.7 - assert result.iloc[0]["GPU [%]"] == 99.9 - assert result.iloc[0]["VRAM [GiB]"] == 23.8 - - @patch("scitex.resource._get_processor_usages._get_gpu_usage") - @patch("scitex.resource._get_processor_usages._get_cpu_usage") - def test_rounding_behavior(self, mock_cpu, mock_gpu): - """Test DataFrame rounding to 1 decimal place.""" - # The individual functions should return already-rounded values - # since they have their own rounding logic - mock_cpu.return_value = (25.3, 8.2) - mock_gpu.return_value = (65.1, 4.6) - - result = get_processor_usages() - - assert result.iloc[0]["CPU [%]"] == 25.3 - assert result.iloc[0]["RAM [GiB]"] == 8.2 - assert result.iloc[0]["GPU [%]"] == 65.1 - assert result.iloc[0]["VRAM [GiB]"] == 4.6 - - @patch("scitex.resource._get_processor_usages._get_cpu_usage") - def test_cpu_error_handling(self, mock_cpu): - """Test error handling when CPU monitoring fails.""" - mock_cpu.side_effect = RuntimeError("CPU monitoring failed") - - with pytest.raises(RuntimeError, match="Failed to get resource usage"): - get_processor_usages() - - @patch("scitex.resource._get_processor_usages._get_gpu_usage") - @patch("scitex.resource._get_processor_usages._get_cpu_usage") - def test_gpu_error_handling(self, mock_cpu, mock_gpu): - """Test error handling when GPU monitoring fails.""" - mock_cpu.return_value = (25.0, 8.0) - mock_gpu.side_effect = RuntimeError("GPU monitoring failed") - - with pytest.raises(RuntimeError, match="Failed to get resource usage"): - get_processor_usages() - - -class TestGetCpuUsage: - """Test suite for _get_cpu_usage function.""" - - @patch("scitex.resource._get_processor_usages.psutil") - def test_basic_cpu_usage(self, mock_psutil): - """Test basic CPU and RAM usage retrieval.""" - mock_memory = Mock() - mock_memory.percent = 60.0 - mock_memory.total = 16 * (1024**3) # 16 GB - - mock_psutil.cpu_percent.return_value = 45.7 - mock_psutil.virtual_memory.return_value = mock_memory - - cpu_perc, ram_gb = _get_cpu_usage() - - assert cpu_perc == 45.7 - assert ram_gb == 9.6 # 60% of 16 GB - - @patch("scitex.resource._get_processor_usages.psutil") - def test_rounding_precision(self, mock_psutil): - """Test rounding precision control.""" - mock_memory = Mock() - mock_memory.percent = 75.456 - mock_memory.total = 8 * (1024**3) # 8 GB - - mock_psutil.cpu_percent.return_value = 33.789 - mock_psutil.virtual_memory.return_value = mock_memory - - cpu_perc, ram_gb = _get_cpu_usage(n_round=2) - - assert cpu_perc == 33.79 - assert ram_gb == 6.04 # 75.456% of 8 GB, rounded to 2 decimals - - @patch("scitex.resource._get_processor_usages.psutil") - def test_zero_usage(self, mock_psutil): - """Test with zero CPU and RAM usage.""" - mock_memory = Mock() - mock_memory.percent = 0.0 - mock_memory.total = 32 * (1024**3) # 32 GB - - mock_psutil.cpu_percent.return_value = 0.0 - mock_psutil.virtual_memory.return_value = mock_memory - - cpu_perc, ram_gb = _get_cpu_usage() - - assert cpu_perc == 0.0 - assert ram_gb == 0.0 - - @patch("scitex.resource._get_processor_usages.psutil") - def test_max_usage(self, mock_psutil): - """Test with maximum CPU and RAM usage.""" - mock_memory = Mock() - mock_memory.percent = 100.0 - mock_memory.total = 64 * (1024**3) # 64 GB - - mock_psutil.cpu_percent.return_value = 100.0 - mock_psutil.virtual_memory.return_value = mock_memory - - cpu_perc, ram_gb = _get_cpu_usage() - - assert cpu_perc == 100.0 - assert ram_gb == 64.0 - - @patch("scitex.resource._get_processor_usages.psutil") - def test_psutil_error_handling(self, mock_psutil): - """Test error handling for psutil failures.""" - mock_psutil.cpu_percent.side_effect = Exception("psutil error") - - with pytest.raises(RuntimeError, match="Failed to get CPU/RAM usage"): - _get_cpu_usage() - - -class TestGetGpuUsage: - """Test suite for _get_gpu_usage function.""" - - @patch("scitex.resource._get_processor_usages.subprocess.run") - def test_basic_gpu_usage(self, mock_run): - """Test basic GPU and VRAM usage retrieval.""" - mock_result = Mock() - mock_result.stdout = "75,2048" - mock_run.return_value = mock_result - - gpu_perc, vram_gb = _get_gpu_usage() - - assert gpu_perc == 75.0 - assert vram_gb == 2.0 # 2048 MiB = 2.0 GiB - - mock_run.assert_called_once_with( - [ - "nvidia-smi", - "--query-gpu=utilization.gpu,memory.used", - "--format=csv,nounits,noheader", - ], - capture_output=True, - text=True, - check=True, - ) - - @patch("scitex.resource._get_processor_usages.subprocess.run") - def test_zero_gpu_usage(self, mock_run): - """Test with zero GPU usage.""" - mock_result = Mock() - mock_result.stdout = "0,0" - mock_run.return_value = mock_result - - gpu_perc, vram_gb = _get_gpu_usage() - - assert gpu_perc == 0.0 - assert vram_gb == 0.0 - - @patch("scitex.resource._get_processor_usages.subprocess.run") - def test_high_gpu_usage(self, mock_run): - """Test with high GPU usage.""" - mock_result = Mock() - mock_result.stdout = "99,12288" # 12 GB VRAM - mock_run.return_value = mock_result - - gpu_perc, vram_gb = _get_gpu_usage() - - assert gpu_perc == 99.0 - assert vram_gb == 12.0 - - @patch("scitex.resource._get_processor_usages.subprocess.run") - def test_rounding_precision(self, mock_run): - """Test rounding precision control.""" - mock_result = Mock() - mock_result.stdout = "67,3456" # 3.375 GB VRAM - mock_run.return_value = mock_result - - gpu_perc, vram_gb = _get_gpu_usage(n_round=3) - - assert gpu_perc == 67.0 - assert vram_gb == 3.375 - - @patch("scitex.resource._get_processor_usages.subprocess.run") - def test_nvidia_smi_not_available(self, mock_run): - """Test fallback when nvidia-smi is not available.""" - mock_run.side_effect = subprocess.CalledProcessError(1, "nvidia-smi") - - gpu_perc, vram_gb = _get_gpu_usage() - - assert gpu_perc == 0.0 - assert vram_gb == 0.0 - - @patch("scitex.resource._get_processor_usages.subprocess.run") - def test_invalid_output_format(self, mock_run): - """Test fallback with invalid nvidia-smi output.""" - mock_result = Mock() - mock_result.stdout = "invalid,output,format" - mock_run.return_value = mock_result - - gpu_perc, vram_gb = _get_gpu_usage() - - assert gpu_perc == 0.0 - assert vram_gb == 0.0 - - @patch("scitex.resource._get_processor_usages.subprocess.run") - def test_empty_output(self, mock_run): - """Test fallback with empty nvidia-smi output.""" - mock_result = Mock() - mock_result.stdout = "" - mock_run.return_value = mock_result - - gpu_perc, vram_gb = _get_gpu_usage() - - assert gpu_perc == 0.0 - assert vram_gb == 0.0 - - @patch("scitex.resource._get_processor_usages.subprocess.run") - def test_non_numeric_values(self, mock_run): - """Test fallback with non-numeric nvidia-smi output.""" - mock_result = Mock() - mock_result.stdout = "N/A,N/A" - mock_run.return_value = mock_result - - gpu_perc, vram_gb = _get_gpu_usage() - - assert gpu_perc == 0.0 - assert vram_gb == 0.0 - - -if __name__ == "__main__": - import os - - import pytest - - pytest.main([os.path.abspath(__file__)]) - -# -------------------------------------------------------------------------------- -# Start of Source Code from: /home/ywatanabe/proj/scitex-code/src/scitex/resource/_get_processor_usages.py -# -------------------------------------------------------------------------------- -# #!/usr/bin/env python3 -# # -*- coding: utf-8 -*- -# # Time-stamp: "2024-11-04 16:12:50 (ywatanabe)" -# # File: ./scitex_repo/src/scitex/resource/_get_processor_usages.py -# -# """ -# Functionality: -# * Monitors and records system resource utilization (CPU, RAM, GPU, VRAM) -# Input: -# * None (uses system calls and psutil library) -# Output: -# * DataFrame containing resource usage statistics -# Prerequisites: -# * NVIDIA GPU with nvidia-smi installed -# * psutil package -# """ -# -# import os -# import subprocess -# import sys -# from datetime import datetime -# from typing import Optional, Tuple -# -# import matplotlib.pyplot as plt -# import pandas as pd -# import psutil -# -# -# def get_processor_usages() -> pd.DataFrame: -# """Gets current system resource usage statistics. -# -# Returns -# ------- -# pd.DataFrame -# Resource usage data with columns: -# - Timestamp: Timestamp -# - CPU [%]: CPU utilization -# - RAM [GiB]: RAM usage -# - GPU [%]: GPU utilization -# - VRAM [GiB]: VRAM usage -# -# Example -# ------- -# >>> df = get_proccessor_usages() -# >>> print(df) -# Timestamp CPU [%] RAM [GiB] GPU [%] VRAM [GiB] -# 0 2024-11-04 10:30:15 25.3 8.2 65.0 4.5 -# """ -# try: -# cpu_perc, ram_gb = _get_cpu_usage() -# gpu_perc, vram_gb = _get_gpu_usage() -# -# sr = pd.Series( -# { -# "Timestamp": datetime.now(), -# "CPU [%]": cpu_perc, -# "RAM [GiB]": ram_gb, -# "GPU [%]": gpu_perc, -# "VRAM [GiB]": vram_gb, -# } -# ) -# -# return pd.DataFrame(sr).round(1).T -# except Exception as err: -# raise RuntimeError(f"Failed to get resource usage: {err}") -# -# -# def _get_cpu_usage( -# process: Optional[int] = os.getpid(), n_round: int = 1 -# ) -> Tuple[float, float]: -# """Gets CPU and RAM usage statistics. -# -# Parameters -# ---------- -# process : int, optional -# Process ID to monitor -# n_round : int, optional -# Number of decimal places to round to -# -# Returns -# ------- -# Tuple[float, float] -# CPU usage percentage and RAM usage in GiB -# """ -# try: -# cpu_usage_perc = psutil.cpu_percent() -# ram_usage_gb = ( -# psutil.virtual_memory().percent -# / 100 -# * psutil.virtual_memory().total -# / (1024**3) -# ) -# return round(cpu_usage_perc, n_round), round(ram_usage_gb, n_round) -# except Exception as err: -# raise RuntimeError(f"Failed to get CPU/RAM usage: {err}") -# -# -# def _get_gpu_usage(n_round: int = 1) -> Tuple[float, float]: -# """Gets GPU and VRAM usage statistics. -# -# Parameters -# ---------- -# n_round : int, optional -# Number of decimal places to round to -# -# Returns -# ------- -# Tuple[float, float] -# GPU usage percentage and VRAM usage in GiB -# """ -# try: -# result = subprocess.run( -# [ -# "nvidia-smi", -# "--query-gpu=utilization.gpu,memory.used", -# "--format=csv,nounits,noheader", -# ], -# capture_output=True, -# text=True, -# check=True, -# ) -# gpu_usage_perc, vram_usage_mib = result.stdout.strip().split(",") -# vram_usage_gb = float(vram_usage_mib) / 1024 -# return round(float(gpu_usage_perc), n_round), round(vram_usage_gb, n_round) -# except: -# return 0.0, 0.0 -# # except subprocess.CalledProcessError as err: -# # raise RuntimeError(f"Failed to execute nvidia-smi: {err}") -# # except Exception as err: -# # raise RuntimeError(f"Failed to get GPU/VRAM usage: {err}") -# -# -# # def _get_gpu_usage(n_round: int = 1) -> Tuple[float, float]: -# # """Gets GPU and VRAM usage statistics. -# -# # Parameters -# # ---------- -# # n_round : int, optional -# # Number of decimal places to round to -# -# # Returns -# # ------- -# # Tuple[float, float] -# # GPU usage percentage and VRAM usage in GiB -# # """ -# # try: -# # result = subprocess.run( -# # [ -# # "nvidia-smi", -# # "--query-gpu=utilization.gpu,memory.used", -# # "--format=csv,nounits,noheader", -# # ], -# # capture_output=True, -# # text=True, -# # check=True, -# # ) -# # gpu_usage_perc, vram_usage_mib = result.stdout.strip().split(",") -# # vram_usage_gb = float(vram_usage_mib) / 1024 -# # return round(float(gpu_usage_perc), n_round), round(vram_usage_gb, n_round) -# # except Exception as e: -# # print(e) -# # return 0.0, 0.0 # Return zeros when nvidia-smi is not available -# -# -# if __name__ == "__main__": -# import scitex -# -# CONFIG, sys.stdout, sys.stderr, plt, CC = scitex.session.start( -# sys, plt, verbose=False -# ) -# -# usage = scitex.resource.get_processor_usages() -# scitex.io.save(usage, "usage.csv") -# -# scitex.session.close(CONFIG, verbose=False, notify=False) -# -# # EOF -# -# # #!/usr/bin/env python3 -# # # -*- coding: utf-8 -*- -# # # Time-stamp: "2024-11-04 10:27:35 (ywatanabe)" -# # # File: ./scitex_repo/src/scitex/resource/_get_processor_usages.py -# -# # """ -# # This script does XYZ. -# # """ -# -# # # Functions -# # import os -# # import subprocess -# # import sys -# # from datetime import datetime -# -# # import matplotlib.pyplot as plt -# # import scitex -# # import pandas as pd -# # import psutil -# -# -# # # Functions -# # def get_processor_usages(): -# # """ -# # Retrieves the current usage statistics for the CPU, RAM, GPU, and VRAM. -# -# # This function fetches the current usage percentages for the CPU and GPU, as well as the current usage in GiB for RAM and VRAM. -# # The data is then compiled into a pandas DataFrame with the current timestamp. -# -# # Returns: -# # pd.DataFrame: A pandas DataFrame containing the current usage statistics with the following columns: -# # - Time: The timestamp when the data was retrieved. -# # - CPU [%]: The CPU usage percentage. -# # - RAM [GiB]: The RAM usage in GiB. -# # - GPU [%]: The GPU usage percentage. -# # - VRAM [GiB]: The VRAM usage in GiB. -# # Each row in the DataFrame represents a single instance of data retrieval, rounded to 1 decimal place. -# -# # Example: -# # >>> usage_df = get_processor_usages() -# # >>> print(usage_df) -# # """ -# # cpu_perc, ram_gb = _get_cpu_usage() -# # gpu_perc, vram_gb = _get_gpu_usage() -# -# # sr = pd.Series( -# # { -# # "Time": datetime.now(), -# # "CPU [%]": cpu_perc, -# # "RAM [GiB]": ram_gb, -# # "GPU [%]": gpu_perc, -# # "VRAM [GiB]": vram_gb, -# # } -# # ) -# -# # df = pd.DataFrame(sr).round(1).T -# -# # return df -# -# -# # def _get_cpu_usage(process=os.getpid(), n_round=1): -# # cpu_usage_perc = psutil.cpu_percent() -# # ram_usage_gb = ( -# # psutil.virtual_memory().percent -# # / 100 -# # * psutil.virtual_memory().total -# # / (1024**3) -# # ) -# # return round(cpu_usage_perc, n_round), round(ram_usage_gb, n_round) -# -# -# # def _get_gpu_usage(n_round=1): -# # result = subprocess.run( -# # [ -# # "nvidia-smi", -# # "--query-gpu=utilization.gpu,memory.used", -# # "--format=csv,nounits,noheader", -# # ], -# # capture_output=True, -# # text=True, -# # ) -# # gpu_usage_perc, _vram_usage_mib = result.stdout.strip().split(",") -# # vram_usage_gb = float(_vram_usage_mib) / 1024 -# # return round(float(gpu_usage_perc), n_round), round( -# # float(vram_usage_gb), n_round -# # ) -# -# -# # # (YOUR AWESOME CODE) -# -# # if __name__ == "__main__": -# # # Start -# # CONFIG, sys.stdout, sys.stderr, plt, CC = scitex.session.start( -# # sys, plt, verbose=False -# # ) -# -# # usage = scitex.resource.get_processor_usages() -# # scitex.io.save(usage, "usage.csv") -# -# # # Close -# # scitex.session.close(CONFIG, verbose=False, notify=False) -# -# # -# -# # EOF - -# -------------------------------------------------------------------------------- -# End of Source Code from: /home/ywatanabe/proj/scitex-code/src/scitex/resource/_get_processor_usages.py -# -------------------------------------------------------------------------------- diff --git a/tests/scitex/resource/test__get_specs.py b/tests/scitex/resource/test__get_specs.py deleted file mode 100644 index d340a1ae7..000000000 --- a/tests/scitex/resource/test__get_specs.py +++ /dev/null @@ -1,750 +0,0 @@ -#!/usr/bin/env python3 -# Timestamp: "2025-06-02 16:50:00 (ywatanabe)" -# File: ./tests/scitex/resource/test__get_specs.py - -import platform -from unittest.mock import MagicMock, patch - -import pytest - - -def test_get_specs_default(): - """Test get_specs with default parameters.""" - from scitex.resource import get_specs - - with patch("scitex.resource._get_specs.get_env_info") as mock_env: - mock_env.return_value._asdict.return_value = { - "os": "Linux", - "gcc_version": "9.4.0", - "python_version": "3.8.10", - "torch_version": "1.10.0", - "is_cuda_available": True, - "pip_version": "21.0", - "pip_packages": [], - "conda_packages": [], - "nvidia_gpu_models": "GeForce RTX 3080", - "nvidia_driver_version": "470.103.01", - "cuda_runtime_version": "11.4", - "cudnn_version": "8.2.4", - } - - result = get_specs() - - assert isinstance(result, dict) - assert "Collected Time" in result - assert "System Information" in result - assert "CPU Info" in result - assert "Memory Info" in result - assert "GPU Info" in result - assert "Disk Info" in result - assert "Network Info" in result - - -def test_get_specs_selective_collection(): - """Test get_specs with selective information collection.""" - from scitex.resource import get_specs - - with patch("scitex.resource._get_specs.get_env_info") as mock_env: - mock_env.return_value._asdict.return_value = { - "os": "Linux", - "gcc_version": "9.4.0", - "nvidia_gpu_models": "GeForce RTX 3080", - "nvidia_driver_version": "470.103.01", - "cuda_runtime_version": "11.4", - "cudnn_version": "8.2.4", - } - - result = get_specs(system=True, cpu=False, gpu=False, disk=False, network=False) - - assert "System Information" in result - assert "CPU Info" not in result - assert "Memory Info" not in result - assert "GPU Info" not in result - assert "Disk Info" not in result - assert "Network Info" not in result - - -def test_get_specs_cpu_only(): - """Test get_specs with only CPU information.""" - from scitex.resource import get_specs - - with patch("scitex.resource._get_specs.get_env_info") as mock_env, patch( - "scitex.resource._get_specs._psutil" - ) as mock_psutil: - mock_env.return_value._asdict.return_value = {"os": "Linux"} - - # Mock CPU info - mock_psutil.cpu_count.side_effect = lambda logical=True: 8 if logical else 4 - mock_psutil.cpu_freq.return_value = MagicMock( - max=3600.0, min=800.0, current=2400.0 - ) - mock_psutil.cpu_percent.side_effect = lambda percpu=False, interval=None: ( - [10, 20, 30, 40, 50, 60, 70, 80] if percpu else 45.0 - ) - - # Mock memory info - mock_mem = MagicMock() - mock_mem.total = 16000000000 - mock_mem.available = 8000000000 - mock_mem.used = 8000000000 - mock_mem.percent = 50.0 - mock_psutil.virtual_memory.return_value = mock_mem - - mock_swap = MagicMock() - mock_swap.total = 2000000000 - mock_swap.free = 1000000000 - mock_swap.used = 1000000000 - mock_swap.percent = 50.0 - mock_psutil.swap_memory.return_value = mock_swap - - result = get_specs(system=False, cpu=True, gpu=False, disk=False, network=False) - - assert "CPU Info" in result - assert "Memory Info" in result - assert result["CPU Info"]["Physical cores"] == 4 - assert result["CPU Info"]["Total cores"] == 8 - assert "MHz" in result["CPU Info"]["Max Frequency"] - - -def test_get_specs_yaml_output(): - """Test get_specs with YAML output format.""" - from scitex.resource import get_specs - - mock_supple_info = { - "os": "Linux", - "gcc_version": "9.4.0", - "python_version": "3.8.10", - "torch_version": "1.10.0", - "is_cuda_available": True, - "pip_version": "21.0", - "pip_packages": [], - "conda_packages": [], - "nvidia_gpu_models": "GeForce RTX 3080", - "nvidia_driver_version": "470.103.01", - "cuda_runtime_version": "11.4", - "cudnn_version": "8.2.4", - } - - with patch("scitex.resource._get_specs.get_env_info") as mock_env, patch( - "scitex.resource._get_specs._yaml" - ) as mock_yaml: - mock_env.return_value._asdict.return_value = mock_supple_info - mock_yaml.dump.return_value = "yaml_string_output" - - result = get_specs( - system=True, cpu=False, gpu=False, disk=False, network=False, yaml=True - ) - - assert result == "yaml_string_output" - mock_yaml.dump.assert_called_once() - - -def test_get_specs_verbose_output(capsys): - """Test get_specs with verbose output.""" - from scitex.resource import get_specs - - mock_supple_info = { - "os": "Linux", - "gcc_version": "9.4.0", - "python_version": "3.8.10", - "torch_version": "1.10.0", - "is_cuda_available": True, - "pip_version": "21.0", - "pip_packages": [], - "conda_packages": [], - "nvidia_gpu_models": "GeForce RTX 3080", - "nvidia_driver_version": "470.103.01", - "cuda_runtime_version": "11.4", - "cudnn_version": "8.2.4", - } - - with patch("scitex.resource._get_specs.get_env_info") as mock_env: - mock_env.return_value._asdict.return_value = mock_supple_info - - result = get_specs( - system=True, cpu=False, gpu=False, disk=False, network=False, verbose=True - ) - - captured = capsys.readouterr() - assert len(captured.out) > 0 # Should have printed something - assert isinstance(result, dict) - - -def test_system_info(): - """Test _system_info function.""" - from scitex.resource import _system_info - - with patch("scitex.resource._get_specs._platform") as mock_platform, patch( - "scitex.resource._get_specs._supple_os_info" - ) as mock_os_info: - mock_uname = MagicMock() - mock_uname.node = "test-node" - mock_uname.release = "5.4.0" - mock_uname.version = "#42-Ubuntu" - mock_platform.uname.return_value = mock_uname - - mock_os_info.return_value = {"os": "Ubuntu 20.04"} - - result = _system_info() - - assert result["OS"] == "Ubuntu 20.04" - assert result["Node Name"] == "test-node" - assert result["Release"] == "5.4.0" - assert result["Version"] == "#42-Ubuntu" - - -def test_cpu_info(): - """Test _cpu_info function.""" - from scitex.resource import _cpu_info - - with patch("scitex.resource._get_specs._psutil") as mock_psutil: - mock_psutil.cpu_count.side_effect = lambda logical=True: 8 if logical else 4 - mock_psutil.cpu_freq.return_value = MagicMock( - max=3600.0, min=800.0, current=2400.0 - ) - mock_psutil.cpu_percent.side_effect = lambda percpu=False, interval=None: ( - [10, 20, 30, 40] if percpu else 25.0 - ) - - result = _cpu_info() - - assert result["Physical cores"] == 4 - assert result["Total cores"] == 8 - assert result["Max Frequency"] == "3600.00 MHz" - assert result["Min Frequency"] == "800.00 MHz" - assert result["Current Frequency"] == "2400.00 MHz" - assert result["Total CPU Usage"] == "25.0%" - assert len(result["CPU Usage Per Core"]) == 4 - - -def test_memory_info(): - """Test _memory_info function.""" - from scitex.resource import _memory_info - - with patch("scitex.resource._get_specs._psutil") as mock_psutil: - mock_mem = MagicMock() - mock_mem.total = 16000000000 # 16GB - mock_mem.available = 8000000000 # 8GB - mock_mem.used = 8000000000 # 8GB - mock_mem.percent = 50.0 - mock_psutil.virtual_memory.return_value = mock_mem - - mock_swap = MagicMock() - mock_swap.total = 2000000000 # 2GB - mock_swap.free = 1000000000 # 1GB - mock_swap.used = 1000000000 # 1GB - mock_swap.percent = 50.0 - mock_psutil.swap_memory.return_value = mock_swap - - result = _memory_info() - - assert "Memory" in result - assert "SWAP" in result - assert result["Memory"]["Percentage"] == 50.0 - assert result["SWAP"]["Percentage"] == 50.0 - # Check for byte units (GB/GiB/MB/MiB) - total_str = result["Memory"]["Total"] - assert any(unit in total_str for unit in ["GB", "GiB", "MB", "MiB"]) - - -def test_disk_info(): - """Test _disk_info function.""" - from scitex.resource import _disk_info - - with patch("scitex.resource._get_specs._psutil") as mock_psutil: - # Mock partition - mock_partition = MagicMock() - mock_partition.device = "/dev/sda1" - mock_partition.mountpoint = "/" - mock_partition.fstype = "ext4" - mock_psutil.disk_partitions.return_value = [mock_partition] - - # Mock disk usage - mock_usage = MagicMock() - mock_usage.total = 1000000000000 # 1TB - mock_usage.used = 500000000000 # 500GB - mock_usage.free = 500000000000 # 500GB - mock_usage.percent = 50.0 - mock_psutil.disk_usage.return_value = mock_usage - - # Mock disk I/O - mock_io = MagicMock() - mock_io.read_bytes = 1000000000 # 1GB - mock_io.write_bytes = 500000000 # 500MB - mock_psutil.disk_io_counters.return_value = mock_io - - result = _disk_info() - - assert "Partitions" in result - assert "Total read" in result - assert "Total write" in result - assert "/dev/sda1" in result["Partitions"] - assert result["Partitions"]["/dev/sda1"]["File system type"] == "ext4" - assert result["Partitions"]["/dev/sda1"]["Percentage"] == 50.0 - - -def test_disk_info_permission_error(): - """Test _disk_info function with permission error.""" - from scitex.resource import _disk_info - - with patch("scitex.resource._get_specs._psutil") as mock_psutil: - # Mock partition - mock_partition = MagicMock() - mock_partition.device = "/dev/sda1" - mock_partition.mountpoint = "/restricted" - mock_partition.fstype = "ext4" - mock_psutil.disk_partitions.return_value = [mock_partition] - - # Mock permission error - mock_psutil.disk_usage.side_effect = PermissionError("Access denied") - - # Mock disk I/O - mock_io = MagicMock() - mock_io.read_bytes = 1000000000 - mock_io.write_bytes = 500000000 - mock_psutil.disk_io_counters.return_value = mock_io - - result = _disk_info() - - # Should handle permission error gracefully - assert "Partitions" in result - assert len(result["Partitions"]) == 0 # No accessible partitions - - -def test_network_info(): - """Test _network_info function.""" - from scitex.resource import _network_info - - with patch("scitex.resource._get_specs._psutil") as mock_psutil: - # Mock network interfaces - mock_address = MagicMock() - mock_address.address = "192.168.1.100" - mock_address.netmask = "255.255.255.0" - mock_address.broadcast = "192.168.1.255" - - mock_interfaces = {"eth0": [mock_address], "lo": [mock_address]} - mock_psutil.net_if_addrs.return_value = mock_interfaces - - # Mock network I/O - mock_io = MagicMock() - mock_io.bytes_sent = 1000000000 # 1GB - mock_io.bytes_recv = 2000000000 # 2GB - mock_psutil.net_io_counters.return_value = mock_io - - result = _network_info() - - assert "Interfaces" in result - assert "Total Sent" in result - assert "Total Received" in result - assert "eth0" in result["Interfaces"] - assert "lo" in result["Interfaces"] - assert result["Interfaces"]["eth0"][0]["Address"] == "192.168.1.100" - - -def test_supple_os_info(): - """Test _supple_os_info function.""" - from scitex.resource import _supple_os_info - - with patch( - "scitex.resource._get_specs._SUPPLE_INFO", - {"os": "Ubuntu 20.04", "gcc_version": "9.4.0", "other_key": "other_value"}, - ): - result = _supple_os_info() - - assert result["os"] == "Ubuntu 20.04" - assert result["gcc_version"] == "9.4.0" - assert "other_key" not in result # Should only include specified keys - - -def test_supple_python_info(): - """Test _supple_python_info function.""" - from scitex.resource import _supple_python_info - - with patch( - "scitex.resource._get_specs._SUPPLE_INFO", - { - "python_version": "3.8.10", - "torch_version": "1.10.0", - "is_cuda_available": True, - "pip_version": "21.0", - "pip_packages": ["numpy", "pandas"], - "conda_packages": ["pytorch"], - "other_key": "other_value", - }, - ): - result = _supple_python_info() - - assert result["python_version"] == "3.8.10" - assert result["torch_version"] == "1.10.0" - assert result["is_cuda_available"] is True - assert result["pip_packages"] == ["numpy", "pandas"] - assert "other_key" not in result - - -def test_supple_nvidia_info(): - """Test _supple_nvidia_info function.""" - from scitex.resource import _supple_nvidia_info - - with patch( - "scitex.resource._get_specs._SUPPLE_INFO", - { - "nvidia_gpu_models": "GeForce RTX 3080", - "nvidia_driver_version": "470.103.01", - "cuda_runtime_version": "11.4", - "cudnn_version": "8.2.4", - "other_key": "other_value", - }, - ): - result = _supple_nvidia_info() - - assert "NVIDIA GPU models" in result - assert "NVIDIA Driver version" in result - assert "CUDA Runtime version" in result - assert "cuDNN version" in result - assert "other_key" not in result - - assert result["NVIDIA GPU models"] == "GeForce RTX 3080" - assert result["NVIDIA Driver version"] == "470.103.01" - - -def test_get_specs_integration(): - """Test get_specs integration with real system calls (minimal).""" - from scitex.resource import get_specs - - # Test that function doesn't crash with minimal real system data - try: - result = get_specs(system=True, cpu=False, gpu=False, disk=False, network=False) - assert isinstance(result, dict) - assert "Collected Time" in result - # Don't assert specific values since they depend on the test environment - except Exception as e: - pytest.skip(f"Integration test skipped due to system limitations: {e}") - - -def test_collected_time_format(): - """Test that collected time has correct format.""" - import re - - from scitex.resource import get_specs - - with patch("scitex.resource._get_specs.get_env_info") as mock_env: - mock_env.return_value._asdict.return_value = {"os": "Linux"} - - result = get_specs( - system=False, cpu=False, gpu=False, disk=False, network=False - ) - - time_pattern = r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}" - assert re.match(time_pattern, result["Collected Time"]) - - -def test_error_handling_in_subsystems(): - """Test error handling in various subsystems.""" - from scitex.resource import get_specs - - with patch("scitex.resource._get_specs.get_env_info") as mock_env, patch( - "scitex.resource._get_specs._system_info" - ) as mock_system, patch("scitex.resource._get_specs._cpu_info") as mock_cpu: - mock_env.return_value._asdict.return_value = {"os": "Linux"} - mock_system.side_effect = Exception("System info error") - mock_cpu.side_effect = Exception("CPU info error") - - # Should handle errors gracefully and still return basic info - with pytest.raises(Exception): - get_specs(system=True, cpu=True, gpu=False, disk=False, network=False) - - -if __name__ == "__main__": - import os - - import pytest - - pytest.main([os.path.abspath(__file__)]) - -# -------------------------------------------------------------------------------- -# Start of Source Code from: /home/ywatanabe/proj/scitex-code/src/scitex/resource/_get_specs.py -# -------------------------------------------------------------------------------- -# #!/usr/bin/env python3 -# # -*- coding: utf-8 -*- -# # Time-stamp: "2024-11-04 14:16:49 (ywatanabe)" -# # File: ./scitex_repo/src/scitex/resource/_get_specs.py -# -# """ -# This script provides detailed system information including system basics, boot time, CPU, memory, disk, network, and custom user environment variables. -# """ -# -# import platform as _platform -# import sys -# from datetime import datetime as _datetime -# from pprint import pprint -# -# import matplotlib.pyplot as plt -# import psutil as _psutil -# import yaml as _yaml -# from ._utils._get_env_info import get_env_info -# from scitex.str import readable_bytes -# -# -# def get_specs( -# system=True, -# # boot_time=True, -# cpu=True, -# gpu=True, -# disk=True, -# network=True, -# verbose=False, -# yaml=False, -# ): -# """ -# Collects and returns system specifications including system information, CPU, GPU, disk, and network details. -# -# This function gathers various pieces of system information based on the parameters provided. It can return the data in a dictionary format or print it out based on the verbose flag. Additionally, there's an option to format the output as YAML. -# -# Arguments: -# system (bool): If True, collects system-wide information such as OS and node name. Default is True. -# boot_time (bool): If True, collects system boot time. Currently commented out in the implementation. Default is True. -# cpu (bool): If True, collects CPU-specific information including frequency and usage. Default is True. -# gpu (bool): If True, collects GPU-specific information. Default is True. -# disk (bool): If True, collects disk usage information for all partitions. Default is True. -# network (bool): If True, collects network interface and traffic information. Default is True. -# verbose (bool): If True, prints the collected information using pprint. Default is False. -# yaml (bool): If True, formats the collected information as YAML. This modifies the return type to a YAML formatted string. Default is False. -# -# Returns: -# dict or str: By default, returns a dictionary containing the collected system specifications. If `yaml` is True, returns a YAML-formatted string instead. -# -# Note: -# - The actual collection of system, CPU, GPU, disk, and network information depends on the availability of corresponding libraries and access permissions. -# - The `boot_time` argument is currently not used as its corresponding code is commented out. -# - The function uses global variables and imports within its scope, which might affect its reusability and testability. -# -# Example: -# >>> specs = get_specs(verbose=True) -# This will print and return the system specifications based on the default parameters. -# -# Dependencies: -# - This function depends on the `scitex` library for accessing system information and formatting output. Ensure this library is installed and properly configured. -# - Python standard libraries: `datetime`, `platform`, `psutil`, `yaml` (optional for YAML output). -# -# Raises: -# PermissionError: If the function lacks necessary permissions to access certain system information, especially disk and network details. -# """ -# -# # To prevent import errors, _SUPPLE_INFO is collected here. -# global _SUPPLE_INFO -# _SUPPLE_INFO = get_env_info()._asdict() -# -# collected_info = {} # OrderedDict() -# -# collected_info["Collected Time"] = _datetime.now().strftime("%Y-%m-%d %H:%M:%S") -# if system: -# collected_info["System Information"] = _system_info() -# # if boot_time: -# # collected_info["Boot Time"] = _boot_time_info() -# if cpu: -# collected_info["CPU Info"] = _cpu_info() -# collected_info["Memory Info"] = _memory_info() -# if gpu: -# collected_info["GPU Info"] = _supple_nvidia_info() -# # scitex.gen.placeholder() -# if disk: -# collected_info["Disk Info"] = _disk_info() -# if network: -# collected_info["Network Info"] = _network_info() -# -# if yaml: -# collected_info = _yaml.dump(collected_info, sort_keys=False) -# -# if verbose: -# pprint(collected_info) -# -# return collected_info -# -# -# def _system_info(): -# uname = _platform.uname() -# return { -# "OS": _supple_os_info()["os"], -# # "GCC version": _supple_os_info()["gcc_version"], -# # "System": uname.system, -# "Node Name": uname.node, -# "Release": uname.release, -# "Version": uname.version, -# # "Machine": uname.machine, -# # "Processor": uname.processor, -# } -# -# -# # def _boot_time_info(): -# # boot_time_timestamp = _psutil.boot_time() -# # bt = _datetime.fromtimestamp(boot_time_timestamp) -# # return { -# # "Boot Time": f"{bt.year}-{bt.month:02d}-{bt.day:02d} {bt.hour:02d}:{bt.minute:02d}:{bt.second:02d}" -# # } -# -# -# def _cpu_info(): -# cpufreq = _psutil.cpu_freq() -# cpu_usage_per_core = _psutil.cpu_percent(percpu=True, interval=1) -# return { -# "Physical cores": _psutil.cpu_count(logical=False), -# "Total cores": _psutil.cpu_count(logical=True), -# "Max Frequency": f"{cpufreq.max:.2f} MHz", -# "Min Frequency": f"{cpufreq.min:.2f} MHz", -# "Current Frequency": f"{cpufreq.current:.2f} MHz", -# "CPU Usage Per Core": { -# f"Core {i}": f"{percentage}%" -# for i, percentage in enumerate(cpu_usage_per_core) -# }, -# "Total CPU Usage": f"{_psutil.cpu_percent()}%", -# } -# -# -# def _memory_info(): -# import scitex -# -# svmem = _psutil.virtual_memory() -# swap = _psutil.swap_memory() -# -# return { -# "Memory": { -# "Total": readable_bytes(svmem.total), -# "Available": readable_bytes(svmem.available), -# "Used": readable_bytes(svmem.used), -# "Percentage": svmem.percent, -# }, -# "SWAP": { -# "Total": readable_bytes(swap.total), -# "Free": readable_bytes(swap.free), -# "Used": readable_bytes(swap.used), -# "Percentage": swap.percent, -# }, -# } -# -# -# def _disk_info(): -# import scitex -# -# partitions_info = {} -# partitions = _psutil.disk_partitions() -# for partition in partitions: -# try: -# usage = _psutil.disk_usage(partition.mountpoint) -# partitions_info[partition.device] = { -# "Mountpoint": partition.mountpoint, -# "File system type": partition.fstype, -# "Total Size": readable_bytes(usage.total), -# "Used": readable_bytes(usage.used), -# "Free": readable_bytes(usage.free), -# "Percentage": usage.percent, -# } -# except PermissionError: -# continue -# -# disk_io = _psutil.disk_io_counters() -# return { -# "Partitions": partitions_info, -# "Total read": readable_bytes(disk_io.read_bytes), -# "Total write": readable_bytes(disk_io.write_bytes), -# } -# -# -# def _network_info(): -# import scitex -# -# if_addrs = _psutil.net_if_addrs() -# interfaces = {} -# for interface_name, interface_addresses in if_addrs.items(): -# interface_info = [] -# for address in interface_addresses: -# interface_info.append( -# { -# # "Address Type": "IP" if address.family == _psutil.AF_INET else "MAC", -# "Address": address.address, -# "Netmask": address.netmask, -# "Broadcast": address.broadcast, -# } -# ) -# interfaces[interface_name] = interface_info -# -# net_io = _psutil.net_io_counters() -# return { -# "Interfaces": interfaces, -# "Total Sent": readable_bytes(net_io.bytes_sent), -# "Total Received": readable_bytes(net_io.bytes_recv), -# } -# -# -# def _python_info(): -# return _supple_python_info() -# -# -# def _supple_os_info(): -# _SUPPLE_OS_KEYS = [ -# "os", -# "gcc_version", -# ] -# return {k: _SUPPLE_INFO[k] for k in _SUPPLE_OS_KEYS} -# -# -# def _supple_python_info(): -# _SUPPLE_PYTHON_KEYS = [ -# "python_version", -# "torch_version", -# "is_cuda_available", -# "pip_version", -# "pip_packages", -# "conda_packages", -# ] -# -# return {k: _SUPPLE_INFO[k] for k in _SUPPLE_PYTHON_KEYS} -# -# -# def _supple_nvidia_info(): -# _SUPPLE_NVIDIA_KEYS = [ -# "nvidia_gpu_models", -# "nvidia_driver_version", -# "cuda_runtime_version", -# "cudnn_version", -# ] -# -# def replace_key(key): -# return key -# -# def replace_key(key): -# key = key.replace("_", " ") -# key = key.replace("nvidia", "NVIDIA") -# key = key.replace("gpu", "GPU") -# key = key.replace("cuda", "CUDA") -# key = key.replace("cudnn", "cuDNN") -# key = key.replace("driver", "Driver") -# key = key.replace("runtime", "Runtime") -# return key -# -# return {replace_key(k): _SUPPLE_INFO[k] for k in _SUPPLE_NVIDIA_KEYS} -# -# -# if __name__ == "__main__": -# import scitex -# -# # Start -# CONFIG, sys.stdout, sys.stderr, plt, CC = scitex.session.start(sys, plt) -# -# info = scitex.res.get_specs() -# scitex.io.save(info, "specs.yaml") -# -# # Close -# scitex.session.close(CONFIG) -# -# # EOF -# -# """ -# /home/ywatanabe/proj/entrance/scitex/res/_get_specs.py -# """ -# -# -# # EOF - -# -------------------------------------------------------------------------------- -# End of Source Code from: /home/ywatanabe/proj/scitex-code/src/scitex/resource/_get_specs.py -# -------------------------------------------------------------------------------- diff --git a/tests/scitex/resource/test__log_processor_usages.py b/tests/scitex/resource/test__log_processor_usages.py deleted file mode 100644 index a4da7150d..000000000 --- a/tests/scitex/resource/test__log_processor_usages.py +++ /dev/null @@ -1,588 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Timestamp: "2025-06-02 15:00:00 (ywatanabe)" -# File: ./tests/scitex/resource/test__log_processor_usages.py - -"""Tests for processor usage logging functionality.""" - -import math -import os -import tempfile -import time -from multiprocessing import Process -from unittest.mock import MagicMock, Mock, mock_open, patch - -import pandas as pd -import pytest - -pytest.importorskip("zarr") - -from scitex.resource import log_processor_usages -from scitex.resource._log_processor_usages import ( - _add, - _ensure_log_file, - _log_processor_usages, -) - - -class TestLogProcessorUsages: - """Test suite for log_processor_usages function.""" - - @patch("scitex.resource._log_processor_usages._log_processor_usages") - def test_foreground_execution(self, mock_log): - """Test foreground execution mode.""" - mock_log.return_value = None - - result = log_processor_usages( - path="/tmp/test.csv", limit_min=1, interval_s=0.1, background=False - ) - - assert result is None - mock_log.assert_called_once_with( - path="/tmp/test.csv", limit_min=1, interval_s=0.1, init=True, verbose=False - ) - - @patch("scitex.resource._log_processor_usages.Process") - def test_background_execution(self, mock_process_class): - """Test background execution mode.""" - mock_process = Mock() - mock_process_class.return_value = mock_process - - result = log_processor_usages( - path="/tmp/test.csv", limit_min=5, background=True - ) - - assert result == mock_process - mock_process_class.assert_called_once() - mock_process.start.assert_called_once() - - @patch("scitex.resource._log_processor_usages._log_processor_usages") - def test_default_parameters(self, mock_log): - """Test with default parameters.""" - mock_log.return_value = None - - log_processor_usages() - - mock_log.assert_called_once_with( - path="/tmp/scitex/processor_usages.csv", - limit_min=30, - interval_s=1, - init=True, - verbose=False, - ) - - @patch("scitex.resource._log_processor_usages._log_processor_usages") - def test_custom_parameters(self, mock_log): - """Test with custom parameters.""" - mock_log.return_value = None - - log_processor_usages( - path="/custom/path.csv", - limit_min=10, - interval_s=2.5, - init=False, - verbose=True, - background=False, - ) - - mock_log.assert_called_once_with( - path="/custom/path.csv", - limit_min=10, - interval_s=2.5, - init=False, - verbose=True, - ) - - -class TestLogProcessorUsagesInternal: - """Test suite for _log_processor_usages function.""" - - @patch("scitex.resource._log_processor_usages.time.sleep") - @patch("scitex.resource._log_processor_usages._add") - @patch("scitex.resource._log_processor_usages._ensure_log_file") - def test_basic_logging(self, mock_ensure, mock_add, mock_sleep): - """Test basic logging functionality.""" - _log_processor_usages( - path="/tmp/test.csv", - limit_min=0.05, # 3 seconds - interval_s=1.0, - init=True, - verbose=False, - ) - - mock_ensure.assert_called_once_with("/tmp/test.csv", True) - assert mock_add.call_count == 3 # ceil(3/1) = 3 calls - assert mock_sleep.call_count == 3 - - # Check sleep was called with correct interval - for call in mock_sleep.call_args_list: - assert call[0][0] == 1.0 - - @patch("scitex.resource._log_processor_usages.time.sleep") - @patch("scitex.resource._log_processor_usages._add") - @patch("scitex.resource._log_processor_usages._ensure_log_file") - def test_timing_calculation(self, mock_ensure, mock_add, mock_sleep): - """Test timing calculation with different intervals.""" - _log_processor_usages( - path="/tmp/test.csv", - limit_min=0.05, # 3 seconds - interval_s=1.5, - init=False, - verbose=True, - ) - - expected_calls = math.ceil(3 / 1.5) # ceil(2.0) = 2 - assert mock_add.call_count == expected_calls - assert mock_sleep.call_count == expected_calls - - def test_csv_path_validation(self): - """Test CSV path validation.""" - with pytest.raises(AssertionError, match="Path must end with .csv"): - _log_processor_usages(path="/tmp/test.txt") - - @patch("scitex.resource._log_processor_usages.time.sleep") - @patch("scitex.resource._log_processor_usages._add") - @patch("scitex.resource._log_processor_usages._ensure_log_file") - def test_verbose_parameter(self, mock_ensure, mock_add, mock_sleep): - """Test verbose parameter is passed to _add.""" - _log_processor_usages( - path="/tmp/test.csv", - limit_min=0.03, # 2 seconds - interval_s=1.0, - verbose=True, - ) - - # Check verbose=True was passed to each _add call - for call in mock_add.call_args_list: - assert call[1]["verbose"] is True - - -class TestEnsureLogFile: - """Test suite for _ensure_log_file function.""" - - @patch("scitex.resource._log_processor_usages.os.path.exists") - @patch("scitex.resource._log_processor_usages.os.makedirs") - @patch("scitex.resource._log_processor_usages.pd.DataFrame") - @patch("scitex.resource._log_processor_usages.printc") - def test_create_new_file(self, mock_printc, mock_df, mock_makedirs, mock_exists): - """Test creating new log file.""" - mock_exists.return_value = False - mock_df_instance = Mock() - mock_df.return_value = mock_df_instance - - _ensure_log_file("/tmp/new/test.csv", init=True) - - mock_makedirs.assert_called_once_with("/tmp/new", exist_ok=True) - mock_df.assert_called_once_with( - columns=["Timestamp", "CPU [%]", "RAM [GiB]", "GPU [%]", "VRAM [GiB]"] - ) - mock_df_instance.to_csv.assert_called_once_with( - "/tmp/new/test.csv", index=False - ) - mock_printc.assert_called_once_with("/tmp/new/test.csv created.") - - @patch("scitex.resource._log_processor_usages.os.path.exists") - @patch("scitex.resource._log_processor_usages.sh") - @patch("scitex.resource._log_processor_usages.os.makedirs") - @patch("scitex.resource._log_processor_usages.pd.DataFrame") - @patch("scitex.resource._log_processor_usages.printc") - def test_reinitialize_existing_file( - self, mock_printc, mock_df, mock_makedirs, mock_sh, mock_exists - ): - """Test reinitializing existing log file.""" - mock_exists.return_value = True - mock_df_instance = Mock() - mock_df.return_value = mock_df_instance - - _ensure_log_file("/tmp/existing.csv", init=True) - - mock_sh.assert_called_once_with("rm -f /tmp/existing.csv") - mock_makedirs.assert_called_once_with("/tmp", exist_ok=True) - mock_df.assert_called_once() - mock_df_instance.to_csv.assert_called_once_with( - "/tmp/existing.csv", index=False - ) - - @patch("scitex.resource._log_processor_usages.os.path.exists") - @patch("scitex.resource._log_processor_usages.sh") - @patch("scitex.resource._log_processor_usages.os.makedirs") - @patch("scitex.resource._log_processor_usages.pd.DataFrame") - def test_keep_existing_file(self, mock_df, mock_makedirs, mock_sh, mock_exists): - """Test keeping existing log file when init=False.""" - mock_exists.return_value = True - - _ensure_log_file("/tmp/existing.csv", init=False) - - mock_sh.assert_not_called() - mock_makedirs.assert_not_called() - mock_df.assert_not_called() - - @patch("scitex.resource._log_processor_usages.os.path.exists") - @patch("scitex.resource._log_processor_usages.sh") - def test_file_removal_error(self, mock_sh, mock_exists): - """Test error handling during file removal.""" - mock_exists.return_value = True - mock_sh.side_effect = Exception("Permission denied") - - with pytest.raises(RuntimeError, match="Failed to init log file"): - _ensure_log_file("/tmp/test.csv", init=True) - - -class TestAddFunction: - """Test suite for _add function.""" - - @patch("scitex.resource._log_processor_usages.get_processor_usages") - @patch("builtins.open", new_callable=mock_open) - def test_basic_append(self, mock_file, mock_get_usage): - """Test basic data appending.""" - # Mock processor usage data - mock_df = Mock() - mock_get_usage.return_value = mock_df - - # Mock file position (empty file) - mock_file.return_value.tell.return_value = 0 - - _add("/tmp/test.csv", verbose=True) - - mock_get_usage.assert_called_once() - mock_file.assert_called_once_with("/tmp/test.csv", "a") - mock_df.to_csv.assert_called_once_with( - mock_file.return_value, header=True, index=False - ) - - @patch("scitex.resource._log_processor_usages.get_processor_usages") - @patch("builtins.open", new_callable=mock_open) - def test_append_to_existing(self, mock_file, mock_get_usage): - """Test appending to existing file (no header).""" - mock_df = Mock() - mock_get_usage.return_value = mock_df - - # Mock file position (non-empty file) - mock_file.return_value.tell.return_value = 100 - - _add("/tmp/test.csv", verbose=False) - - mock_df.to_csv.assert_called_once_with( - mock_file.return_value, header=False, index=False - ) - - @patch("scitex.resource._log_processor_usages.get_processor_usages") - @patch("builtins.open", new_callable=mock_open) - def test_header_logic(self, mock_file, mock_get_usage): - """Test header inclusion logic based on file position.""" - mock_df = Mock() - mock_get_usage.return_value = mock_df - - # Test empty file (header should be included) - mock_file.return_value.tell.return_value = 0 - _add("/tmp/test.csv") - assert mock_df.to_csv.call_args[1]["header"] is True - - # Reset mock - mock_df.reset_mock() - - # Test non-empty file (header should not be included) - mock_file.return_value.tell.return_value = 50 - _add("/tmp/test.csv") - assert mock_df.to_csv.call_args[1]["header"] is False - - @patch("scitex.resource._log_processor_usages.get_processor_usages") - def test_file_error_handling(self, mock_get_usage): - """Test error handling when file cannot be opened.""" - mock_get_usage.return_value = Mock() - - with patch("builtins.open", side_effect=IOError("Cannot open file")): - with pytest.raises(IOError): - _add("/tmp/test.csv") - - -class TestIntegration: - """Integration tests for the logging system.""" - - def test_real_file_operations(self): - """Test with real file operations (using temporary files).""" - with tempfile.TemporaryDirectory() as tmpdir: - log_path = os.path.join(tmpdir, "test_log.csv") - - # Test file creation - _ensure_log_file(log_path, init=True) - assert os.path.exists(log_path) - - # Read the file and check headers - with open(log_path, "r") as f: - content = f.read() - assert "Timestamp,CPU [%],RAM [GiB],GPU [%],VRAM [GiB]" in content - - @patch("scitex.resource._log_processor_usages.get_processor_usages") - def test_csv_format_validation(self, mock_get_usage): - """Test that generated CSV has correct format.""" - # Mock realistic processor usage data - mock_data = pd.DataFrame( - { - "Timestamp": ["2024-01-01 10:00:00"], - "CPU [%]": [25.3], - "RAM [GiB]": [8.2], - "GPU [%]": [65.0], - "VRAM [GiB]": [4.5], - } - ) - mock_get_usage.return_value = mock_data - - with tempfile.TemporaryDirectory() as tmpdir: - log_path = os.path.join(tmpdir, "test_log.csv") - - # Initialize and add data - _ensure_log_file(log_path, init=True) - _add(log_path, verbose=False) - - # Read and validate CSV content - df = pd.read_csv(log_path) - assert len(df) == 1 - assert list(df.columns) == [ - "Timestamp", - "CPU [%]", - "RAM [GiB]", - "GPU [%]", - "VRAM [GiB]", - ] - assert df.iloc[0]["CPU [%]"] == 25.3 - - @patch("scitex.resource._log_processor_usages.time.sleep") - @patch("scitex.resource._log_processor_usages.get_processor_usages") - def test_minimal_logging_session(self, mock_get_usage, mock_sleep): - """Test minimal logging session with mocked time.sleep.""" - mock_data = pd.DataFrame( - { - "Timestamp": ["2024-01-01 10:00:00"], - "CPU [%]": [25.0], - "RAM [GiB]": [8.0], - "GPU [%]": [65.0], - "VRAM [GiB]": [4.0], - } - ) - mock_get_usage.return_value = mock_data - - with tempfile.TemporaryDirectory() as tmpdir: - log_path = os.path.join(tmpdir, "minimal_test.csv") - - # Run very short logging session - _log_processor_usages( - path=log_path, - limit_min=0.03, # ~2 seconds - interval_s=1.0, - init=True, - verbose=False, - ) - - # Verify file exists and has expected entries - assert os.path.exists(log_path) - df = pd.read_csv(log_path) - assert len(df) >= 1 # Should have at least one entry - - -if __name__ == "__main__": - import os - - import pytest - - pytest.main([os.path.abspath(__file__)]) - -# -------------------------------------------------------------------------------- -# Start of Source Code from: /home/ywatanabe/proj/scitex-code/src/scitex/resource/_log_processor_usages.py -# -------------------------------------------------------------------------------- -# #!/usr/bin/env python3 -# # -*- coding: utf-8 -*- -# # Time-stamp: "2024-11-04 16:28:53 (ywatanabe)" -# # File: ./scitex_repo/src/scitex/resource/_log_processor_usages.py -# -# """ -# Functionality: -# * Monitors and logs system resource utilization over time -# Input: -# * Path for saving logs -# * Monitoring duration and interval -# Output: -# * CSV file containing time-series resource usage data -# Prerequisites: -# * scitex package with processor usage monitoring capabilities -# """ -# -# """Imports""" -# import math -# import os -# import sys -# import time -# from multiprocessing import Process -# from typing import Union -# -# import matplotlib.pyplot as plt -# import scitex -# import pandas as pd -# -# from scitex.sh import sh -# from scitex.io._load import load -# from scitex.io._save import save -# from scitex.str import printc -# from ._get_processor_usages import get_processor_usages -# -# """Functions & Classes""" -# -# -# def log_processor_usages( -# path: str = "/tmp/scitex/processor_usages.csv", -# limit_min: float = 30, -# interval_s: float = 1, -# init: bool = True, -# verbose: bool = False, -# background: bool = False, -# ) -> Union[None, Process]: -# """Logs system resource usage over time. -# -# Parameters -# ---------- -# path : str -# Path to save the log file -# limit_min : float -# Monitoring duration in minutes -# interval_s : float -# Sampling interval in seconds -# init : bool -# Whether to clear existing log file -# verbose : bool -# Whether to print the log -# background : bool -# Whether to run in background -# -# Returns -# ------- -# Union[None, Process] -# Process object if background=True, None otherwise -# """ -# if background: -# process = Process( -# target=_log_processor_usages, -# args=(path, limit_min, interval_s, init, verbose), -# ) -# process.start() -# return process -# -# return _log_processor_usages( -# path=path, -# limit_min=limit_min, -# interval_s=interval_s, -# init=init, -# verbose=verbose, -# ) -# -# -# def _log_processor_usages( -# path: str = "/tmp/scitex/processor_usages.csv", -# limit_min: float = 30, -# interval_s: float = 1, -# init: bool = True, -# verbose: bool = False, -# ) -> None: -# """Logs system resource usage over time. -# -# Parameters -# ---------- -# path : str -# Path to save the log file -# limit_min : float -# Monitoring duration in minutes -# interval_s : float -# Sampling interval in seconds -# init : bool -# Whether to clear existing log file -# verbose : bool -# Whether to print the log -# -# Example -# ------- -# >>> log_processor_usages(path="usage_log.csv", limit_min=5) -# """ -# assert path.endswith(".csv"), "Path must end with .csv" -# -# # Log file initialization -# _ensure_log_file(path, init) -# printc(f"Log file can be monitored with with `tail -f {path}`") -# -# limit_s = limit_min * 60 -# n_max = math.ceil(limit_s // interval_s) -# -# for _ in range(n_max): -# _add(path, verbose=verbose) -# time.sleep(interval_s) -# -# -# # def _ensure_log_file(path: str, init: bool) -> None: -# # def _create_path(path): -# # os.makedirs(os.path.dirname(path), exist_ok=True) -# # empty_df = pd.DataFrame() -# # save(empty_df, path, verbose=False) -# # printc(f"{path} created.") -# -# # if not os.path.exists(path): -# # _create_path(path) -# -# # else: -# # if init and os.path.exists(path): -# # try: -# # sh(f"rm -f {path}") -# # _create_path(path) -# # except Exception as err: -# # raise RuntimeError(f"Failed to init log file: {err}") -# -# # def _add(path: str, verbose: bool = True) -> None: -# # past = load(path) -# # now = get_processor_usages() -# -# # combined = pd.concat([past, now]).round(3) -# # save(combined, path, verbose=verbose) -# -# -# def _add(path: str, verbose: bool = True) -> None: -# """Appends current resource usage to CSV file.""" -# now = get_processor_usages() -# -# # Append mode without loading entire file -# with open(path, "a") as f: -# now.to_csv(f, header=f.tell() == 0, index=False) -# -# -# def _ensure_log_file(path: str, init: bool) -> None: -# """Creates or reinitializes log file with headers.""" -# -# def _create_path(path): -# os.makedirs(os.path.dirname(path), exist_ok=True) -# # Write only headers -# headers = ["Timestamp", "CPU [%]", "RAM [GiB]", "GPU [%]", "VRAM [GiB]"] -# pd.DataFrame(columns=headers).to_csv(path, index=False) -# printc(f"{path} created.") -# -# if not os.path.exists(path): -# _create_path(path) -# elif init: -# try: -# sh(f"rm -f {path}") -# _create_path(path) -# except Exception as err: -# raise RuntimeError(f"Failed to init log file: {err}") -# -# -# main = log_processor_usages -# -# if __name__ == "__main__": -# CONFIG, sys.stdout, sys.stderr, plt, CC = scitex.session.start( -# sys, plt, verbose=False -# ) -# main() -# scitex.session.close(CONFIG, verbose=False, notify=False) -# -# # python -c "import scitex; scitex.resource.log_processor_usages(\"/tmp/processor_usages.csv\", init=True)" -# -# # EOF - -# -------------------------------------------------------------------------------- -# End of Source Code from: /home/ywatanabe/proj/scitex-code/src/scitex/resource/_log_processor_usages.py -# -------------------------------------------------------------------------------- diff --git a/tests/scitex/resource/test_limit_ram.py b/tests/scitex/resource/test_limit_ram.py deleted file mode 100644 index 29f01734b..000000000 --- a/tests/scitex/resource/test_limit_ram.py +++ /dev/null @@ -1,236 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Time-stamp: "2025-06-02 15:00:00 (ywatanabe)" -# File: ./tests/scitex/resource/test_limit_ram.py - -import os -import resource -import unittest.mock as mock - -import pytest - - -def test_limit_ram_get_ram_returns_integer(): - """Test that get_ram() returns an integer value.""" - from scitex.resource.limit_ram import get_ram - - # Mock /proc/meminfo content - mock_meminfo = """MemTotal: 8000000 kB -MemFree: 2000000 kB -MemAvailable: 4000000 kB -Buffers: 500000 kB -Cached: 1000000 kB -SwapCached: 0 kB""" - - with mock.patch("builtins.open", mock.mock_open(read_data=mock_meminfo)): - result = get_ram() - - assert isinstance(result, int) - assert result > 0 - - -def test_limit_ram_get_ram_calculation(): - """Test that get_ram() correctly calculates free memory.""" - from scitex.resource.limit_ram import get_ram - - # Mock /proc/meminfo with known values - mock_meminfo = """MemTotal: 8000000 kB -MemFree: 2000000 kB -MemAvailable: 4000000 kB -Buffers: 500000 kB -Cached: 1000000 kB -SwapCached: 0 kB""" - - with mock.patch("builtins.open", mock.mock_open(read_data=mock_meminfo)): - result = get_ram() - - # Expected: MemFree + Buffers + Cached = 2000000 + 500000 + 1000000 = 3500000 - expected = 2000000 + 500000 + 1000000 - assert result == expected - - -def test_limit_ram_get_ram_handles_missing_fields(): - """Test that get_ram() handles missing memory fields gracefully.""" - from scitex.resource.limit_ram import get_ram - - # Mock /proc/meminfo with only some fields - mock_meminfo = """MemTotal: 8000000 kB -MemFree: 2000000 kB -MemAvailable: 4000000 kB -SwapCached: 0 kB""" - - with mock.patch("builtins.open", mock.mock_open(read_data=mock_meminfo)): - result = get_ram() - - # Should only count MemFree (no Buffers or Cached) - assert result == 2000000 - - -def test_limit_ram_get_ram_file_error_handling(): - """Test that get_ram() handles file reading errors.""" - from scitex.resource.limit_ram import get_ram - - with mock.patch("builtins.open", side_effect=FileNotFoundError): - with pytest.raises(FileNotFoundError): - get_ram() - - -@mock.patch("scitex.resource.limit_ram.resource.setrlimit") -@mock.patch("scitex.resource.limit_ram.resource.getrlimit") -@mock.patch("scitex.resource.limit_ram.get_ram") -@mock.patch("builtins.print") -def test_limit_ram_function_basic( - mock_print, mock_get_ram, mock_getrlimit, mock_setrlimit -): - """Test basic functionality of limit_ram() function.""" - from scitex.resource.limit_ram import limit_ram - - # Setup mocks - mock_get_ram.return_value = 4000000 # 4GB in KB - mock_getrlimit.return_value = (8000000 * 1024, 16000000 * 1024) # soft, hard limits - - # This will fail due to missing fmt_size, but that's expected - with pytest.raises(AttributeError): - limit_ram(0.5) - - # Verify get_ram was called before the error - assert mock_get_ram.call_count >= 1 - - # Verify getrlimit was called before the error - mock_getrlimit.assert_called_once_with(resource.RLIMIT_AS) - - # Verify setrlimit was called before the error - mock_setrlimit.assert_called_once() - - -@mock.patch("scitex.resource.limit_ram.resource.setrlimit") -@mock.patch("scitex.resource.limit_ram.resource.getrlimit") -@mock.patch("scitex.resource.limit_ram.get_ram") -def test_limit_ram_function_calculation(mock_get_ram, mock_getrlimit, mock_setrlimit): - """Test that limit_ram() calculates memory limits correctly.""" - from scitex.resource.limit_ram import limit_ram - - # Setup mocks - mock_get_ram.return_value = 4000000 # 4GB in KB - mock_getrlimit.return_value = (8000000 * 1024, 16000000 * 1024) - - # This will fail due to missing fmt_size, but we can still test calculation - with pytest.raises(AttributeError): - limit_ram(0.5) - - # Calculate expected max_val - ram_kb = 4000000 - expected_max_val = min(0.5 * ram_kb * 1024, ram_kb * 1024) - expected_max_val = int(0.5 * ram_kb * 1024) # Should be the smaller value - - # Verify setrlimit was called with correct values before the error - mock_setrlimit.assert_called_once() - call_args = mock_setrlimit.call_args[0][1] - assert call_args[0] == expected_max_val - - -def test_limit_ram_backward_compatibility(): - """Test that deprecated function names still work.""" - from scitex.resource.limit_ram import get_RAM, get_ram, limit_RAM, limit_ram - - # Test that deprecated names reference the same functions - assert limit_RAM is limit_ram - assert get_RAM is get_ram - - -def test_limit_ram_get_ram_edge_cases(): - """Test edge cases for get_ram function.""" - from scitex.resource.limit_ram import get_ram - - # Test with zero values - mock_meminfo_zero = """MemTotal: 8000000 kB -MemFree: 0 kB -MemAvailable: 4000000 kB -Buffers: 0 kB -Cached: 0 kB""" - - with mock.patch("builtins.open", mock.mock_open(read_data=mock_meminfo_zero)): - result = get_ram() - assert result == 0 - - -def test_limit_ram_import_dependencies(): - """Test that all required dependencies can be imported.""" - # Test resource module import - import resource - - assert hasattr(resource, "getrlimit") - assert hasattr(resource, "setrlimit") - assert hasattr(resource, "RLIMIT_AS") - - # Test scitex import - import scitex - - assert hasattr(scitex, "gen") - - -@mock.patch("scitex.resource.limit_ram.resource.setrlimit") -@mock.patch("scitex.resource.limit_ram.resource.getrlimit") -@mock.patch("scitex.resource.limit_ram.get_ram") -def test_limit_ram_resource_error_handling( - mock_get_ram, mock_getrlimit, mock_setrlimit -): - """Test that limit_ram handles resource errors gracefully.""" - from scitex.resource.limit_ram import limit_ram - - mock_get_ram.return_value = 4000000 - mock_getrlimit.return_value = (8000000 * 1024, 16000000 * 1024) - mock_setrlimit.side_effect = OSError("Permission denied") - - # The function will fail with OSError from setrlimit before it gets to fmt_size - with pytest.raises(OSError): - limit_ram(0.5) - - -if __name__ == "__main__": - import os - - import pytest - - pytest.main([os.path.abspath(__file__)]) - -# -------------------------------------------------------------------------------- -# Start of Source Code from: /home/ywatanabe/proj/scitex-code/src/scitex/resource/limit_ram.py -# -------------------------------------------------------------------------------- -# #!/usr/bin/env python3 -# # -*- coding: utf-8 -*- -# # Time-stamp: "2021-09-20 21:02:04 (ywatanabe)" -# -# import resource -# import scitex -# -# -# def limit_ram(ram_factor): -# soft, hard = resource.getrlimit(resource.RLIMIT_AS) -# max_val = min(ram_factor * get_ram() * 1024, get_ram() * 1024) -# resource.setrlimit(resource.RLIMIT_AS, (max_val, hard)) -# print(f"\nFree RAM was limited to {scitex.gen.fmt_size(max_val)}") -# -# -# def get_ram(): -# with open("/proc/meminfo", "r") as mem: -# free_memory = 0 -# for i in mem: -# sline = i.split() -# if str(sline[0]) in ("MemFree:", "Buffers:", "Cached:"): -# free_memory += int(sline[1]) -# return free_memory -# -# -# # Backward compatibility -# limit_RAM = limit_ram # Deprecated: use limit_ram instead -# get_RAM = get_ram # Deprecated: use get_ram instead -# -# -# if __name__ == "__main__": -# get_ram() -# limit_ram(0.1) - -# -------------------------------------------------------------------------------- -# End of Source Code from: /home/ywatanabe/proj/scitex-code/src/scitex/resource/limit_ram.py -# --------------------------------------------------------------------------------