From b0d9551a5bb0bd0e4593ef7f7daab818ab3ea812 Mon Sep 17 00:00:00 2001 From: Yusuke Watanabe Date: Mon, 27 Apr 2026 22:38:17 +0900 Subject: [PATCH 01/10] extract(hpc): bridge scitex.hpc via sys.modules alias to scitex-hpc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scitex_dev.test_runner's HPC dispatch (run_hpc_srun/sbatch/sync/poll/fetch) was generic SLURM code that didn't belong in dev tooling. Extracted to standalone scitex-hpc v0.1.0 package: https://github.com/ywatanabe1989/scitex-hpc Public API in scitex.hpc: - JobConfig (dataclass with SCITEX_HPC_* env-var override resolution) - srun (blocking interactive) - sbatch (async, returns job ID) - sync (rsync local → host) - poll_job (sacct status) - fetch_result (scp .out file back) Login nodes never run compute — every command goes through bash -lc to load SLURM modules, then srun/sbatch. scitex.hpc is scitex_hpc: True (verified) scitex-hpc tests: 12/12 pass --- pyproject.toml | 4 ++++ src/scitex/hpc/__init__.py | 13 +++++++++++++ 2 files changed, 17 insertions(+) create mode 100755 src/scitex/hpc/__init__.py diff --git a/pyproject.toml b/pyproject.toml index bdfecd61..be9b42af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -138,6 +138,10 @@ app = [ "scitex-app>=0.1.0", ] +# HPC Module - Generic SLURM dispatch (srun, sbatch, sync, poll, fetch) +# Use: pip install scitex[hpc] +hpc = ["scitex-hpc>=0.1.0"] + # AI Module - LLM APIs and ML tools # Use: pip install scitex[ai] ai = [ diff --git a/src/scitex/hpc/__init__.py b/src/scitex/hpc/__init__.py new file mode 100755 index 00000000..942df109 --- /dev/null +++ b/src/scitex/hpc/__init__.py @@ -0,0 +1,13 @@ +"""SciTeX hpc — thin compatibility shim for scitex-hpc.""" + +import sys as _sys + +try: + import scitex_hpc as _real +except ImportError as _e: + raise ImportError( + "scitex.hpc requires the 'scitex-hpc' package. " + "Install with: pip install scitex[hpc] (or: pip install scitex-hpc)" + ) from _e + +_sys.modules[__name__] = _real From 223fd6ab602845bbb7974a523098373004aeaa44 Mon Sep 17 00:00:00 2001 From: Yusuke Watanabe Date: Tue, 28 Apr 2026 09:30:32 +0900 Subject: [PATCH 02/10] fix(dsp.hilbert): guard empty-input case (mirrors scitex-dsp fix) --- src/scitex/dsp/_hilbert.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/scitex/dsp/_hilbert.py b/src/scitex/dsp/_hilbert.py index 18108341..6938f7ee 100755 --- a/src/scitex/dsp/_hilbert.py +++ b/src/scitex/dsp/_hilbert.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Time-stamp: "2024-11-04 02:07:11 (ywatanabe)" # File: ./scitex_repo/src/scitex/dsp/_hilbert.py @@ -21,7 +20,15 @@ def hilbert( x, dim=-1, ): - y = Hilbert(x.shape[-1], dim=dim)(x) + seq_len = x.shape[dim] + if seq_len == 0: + empty = ( + x[..., :0] + if dim in (-1, x.ndim - 1) + else x.swapaxes(dim, -1)[..., :0].swapaxes(dim, -1) + ) + return empty, empty + y = Hilbert(seq_len, dim=dim)(x) return y[..., 0], y[..., 1] From c03f9b9a11b9c652b97ba6d782c3bc4fd6d7c60c Mon Sep 17 00:00:00 2001 From: Yusuke Watanabe Date: Tue, 28 Apr 2026 09:35:21 +0900 Subject: [PATCH 03/10] fix(nn.Hilbert): broadcast u along self.dim (mirrors scitex-nn fix) --- src/scitex/nn/_Hilbert.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/scitex/nn/_Hilbert.py b/src/scitex/nn/_Hilbert.py index 14209b46..034fd5fa 100755 --- a/src/scitex/nn/_Hilbert.py +++ b/src/scitex/nn/_Hilbert.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Timestamp: "2025-04-10 12:46:06 (ywatanabe)" # File: /home/ywatanabe/proj/scitex_repo/src/scitex/nn/_Hilbert.py # ---------------------------------------- @@ -42,7 +41,11 @@ def hilbert_transform(self, x): # ) orig_dtype = x.dtype - x = x.float() + # Preserve float32/float64 dtype (float64 stays float64). + # FFT does not support torch.float16/bfloat16 or integer dtypes, + # so only those are upcast to float32; float32/float64 pass through. + if x.dtype not in (torch.float32, torch.float64): + x = x.float() xf = fft(x, n=self.n, dim=self.dim) x = x.to(orig_dtype) @@ -52,6 +55,12 @@ def hilbert_transform(self, x): steepness * self.f.type_as(x) ) # Soft step function for differentiability + # Reshape u to broadcast along self.dim (was implicit -1 only). + if x.ndim > 1: + shape = [1] * x.ndim + shape[self.dim] = self.n + u = u.view(*shape) + transformed = ifft(xf * 2 * u, dim=self.dim) return transformed From 327ad155555ef0d9866f3b02b774f5a1a7c0565b Mon Sep 17 00:00:00 2001 From: Yusuke Watanabe Date: Tue, 28 Apr 2026 11:04:35 +0900 Subject: [PATCH 04/10] =?UTF-8?q?docs(skills/cli):=20add=20=C2=A76c=20Prio?= =?UTF-8?q?rityConfig=20precedence=20+=20=C2=A79=20mutating-ops=20trio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §6c — value-precedence cascade (direct → yaml → env → default) via scitex_config.PriorityConfig. CLI flags always win; do not hand-roll. §9 — observation/dry-run/execute pattern for mutating commands: - Mode flags: default observation, --dry-run preview, --- execute - Flag-naming: name action by scope (--update-hosts not --apply) - --reference names source-of-truth for state-converging ops - Filter flags use plural scope nouns (--hosts, --packages) - Dry-run is enforced via manifest gate (canonical: scitex-dev rename-symbols) Audit checklist updated with the new requirements. --- .../_skills/general/03_interface_02_cli.md | 120 +++++++++++++++++- 1 file changed, 119 insertions(+), 1 deletion(-) diff --git a/src/scitex/_skills/general/03_interface_02_cli.md b/src/scitex/_skills/general/03_interface_02_cli.md index a8099a20..b6f59f89 100644 --- a/src/scitex/_skills/general/03_interface_02_cli.md +++ b/src/scitex/_skills/general/03_interface_02_cli.md @@ -320,6 +320,29 @@ Precedence (highest first): `--config PATH` → `$SCITEX__CONFIG` → `_config.yaml`). Project scope overrides user scope; CLI flags and env vars override both. The full layout rule — two roots, prefix-stripping (`scitex-dev` → `dev`), forbidden locations, `SCITEX_DIR` relocation, `PathManager` usage — lives in `01_arch_06_local-state-directories.md`. Document the fallback order in `--help`. +### 6c. Value precedence — `scitex_config.PriorityConfig` + +For every individual config field (host, port, package list, etc.), resolve via the cascade: + +``` +direct (CLI flag) → config_dict (YAML) → env var → default +``` + +**Direct CLI flags ALWAYS win** — never let the YAML override an explicit `--flag`. Config files come second, env vars third, hardcoded defaults last. + +**Implementation: do not hand-roll the cascade.** Use `scitex_config.PriorityConfig`: + +```python +from scitex_config import PriorityConfig + +cfg = PriorityConfig(config_dict=yaml_data, env_prefix="SCITEX_DEV_") +host = cfg.resolve("host", direct_val=cli_args.host, default="localhost") +``` + +Reference implementations: `scitex_hpc._config.JobConfig.resolve()`, the `scitex_config` package itself. + +**Why centralized:** when every CLI follows the same precedence, operators learn it once. A YAML silently overriding a CLI flag is a class of "I told you to do X but you did Y" bugs that disappear when this rule is enforced uniformly. + ## 7. MCP tool parity When a CLI command has an MCP tool counterpart: @@ -336,7 +359,97 @@ When a CLI command has an MCP tool counterpart: - **Rule:** a user must be able to `cmd --json | jq ...` with zero log contamination on stdout. -## 9. Audit checklist +## 9. Mutating operations — observation → dry-run → execute + +For any command that **changes remote state** (sync, deploy, install, push, delete, etc.), provide three modes selected by mutually exclusive flags. Default = observation. The operator must understand what will happen *before* it happens. + +### 9a. Three modes + +| Mode | Flag | Behavior | +|---|---|---| +| **Observation** (default) | (none) | Read-only audit. Print what's currently true, what's stale, what's missing. Exit 0 if everything matches reference; 1 otherwise. | +| **Dry-run** | `--dry-run` | Print the **exact commands** that would execute (`git pull`, `pip install`, `scp`, etc.) — one per line per (target, item). No execution. | +| **Execute** | `---` (e.g. `--update-hosts`, `--upload-files`) | Actually run the dry-run plan. | + +### 9b. Flag-naming for execute mode + +**Name the action by the scope it touches.** A flag called `--apply` is too abstract — the operator can't tell what gets modified. A flag called `--update-hosts` says "this updates hosts" so the scope is unambiguous. + +Examples: +- `--update-hosts` (sync git + pip on remote machines) +- `--upload-files` (push artifacts somewhere) +- `--delete-stale-runs` (remove old job dirs) + +Avoid: `--apply`, `--commit`, `--go`, `--run` — too generic. + +### 9c. Reference flag for source-of-truth + +When the operation converges to a target state, name the source explicitly: + +``` +--reference origin/develop # default: github branch +--reference localhost # local working copy +--reference pypi # latest published version +--reference : # arbitrary peer +``` + +This makes "what are we converging toward?" answerable from the flags alone. + +### 9d. Filter flags name the object scope + +Filter flags should be **plural nouns** that name the scope: + +| Flag | Scope | +|---|---| +| `--hosts mba,nas` | machines (filter sync targets) | +| `--packages scitex-io,scitex-nn` | packages (filter what to operate on) | +| `--branches main,develop` | git branches | +| `--users alice,bob` | accounts | + +Singular forms (`--host`, `--package`) are accepted as aliases but plurals signal "list values". + +### 9e. Dry-run is **enforced**, not optional + +For destructive or wide-scope mutating operations (renames, mass deletes, cross-host syncs), execute mode must **refuse to run** unless a matching `--dry-run` has been performed recently and the operator confirms the same plan is being executed. + +**Reference implementation: `scitex-dev rename-symbols`** (canonical workflow): + +``` +1. Clean git tree ← refuses if uncommitted changes +2. Dry-run preview ← refuses execute without recent --dry-run +3. Review the change list ← inspect the dry-run file list + counts +4. Real run ← matching --dry-run gates the execute +5. Test the result ← reminder printed after execute +``` + +**Enforcement mechanism:** the dry-run writes a manifest (set of operations + hash of inputs) to a state file (e.g. `~/.scitex//last-dry-run.json`). Execute reads it, recomputes the hash from current state, and refuses to run if: +- No manifest exists, OR +- Manifest is older than N minutes, OR +- Recomputed hash doesn't match (state changed since dry-run) + +The error message tells the operator: "Re-run `--dry-run` first; the plan may have changed." A `--force` escape hatch is provided for CI/scripted contexts where the dry-run was already performed and audited out-of-band. + +**Why enforce, not just suggest:** a non-enforced dry-run gets skipped under time pressure ("I already know what it'll do"). An enforced dry-run is a literal checkpoint where the operator sees the plan, then opts in. This eliminates "I thought it would do X but it did Y" failures. + +### 9f. Worked example + +```bash +# Observation: where do hosts stand vs origin/develop? +scitex-dev ecosystem packages + +# Filter to a subset +scitex-dev ecosystem packages --hosts mba --packages scitex-io + +# Preview what sync would do (writes a manifest) +scitex-dev ecosystem packages --hosts mba --dry-run + +# Execute — refuses if no recent matching --dry-run +scitex-dev ecosystem packages --hosts mba --update-hosts +``` + +The same command surface gives a three-step ritual: observe → preview → act. The operator who reads the dry-run output knows exactly what `--update-hosts` will do, and the tool enforces the order. + +## 10. Audit checklist When auditing a new or existing SciTeX CLI: @@ -347,8 +460,13 @@ When auditing a new or existing SciTeX CLI: - [ ] Deprecated names hard-error with redirect (§5) - [ ] Env vars use `SCITEX__*` prefix (§6a) - [ ] Config file path follows §6b +- [ ] Per-field precedence via `scitex_config.PriorityConfig` (§6c) - [ ] MCP parity if applicable (§7) - [ ] stdout/stderr separation clean (§8) +- [ ] Mutating ops: observation default + `--dry-run` + scope-named execute flag (§9) +- [ ] Filter flags use plural scope nouns (`--hosts`, `--packages`) (§9d) +- [ ] `--reference` names the source-of-truth for state-converging ops (§9c) +- [ ] Destructive ops enforce `--dry-run` before execute via manifest gate (§9e; reference: `scitex-dev rename-symbols`) ## Cross-references From 9d4013d076e40df35e41da02b87f99968749eb27 Mon Sep 17 00:00:00 2001 From: Yusuke Watanabe Date: Tue, 28 Apr 2026 11:10:10 +0900 Subject: [PATCH 05/10] docs(skills/env-vars): cross-link PriorityConfig precedence + cascade order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds 'Resolution precedence' section listing the cascade (direct → yaml → env → default) and pointing to the canonical scitex_config.PriorityConfig implementation. Cross-references 03_interface_02_cli.md §6c. --- .../_skills/general/01_arch_04_environment-variables.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/scitex/_skills/general/01_arch_04_environment-variables.md b/src/scitex/_skills/general/01_arch_04_environment-variables.md index 840665ef..9fbc689e 100644 --- a/src/scitex/_skills/general/01_arch_04_environment-variables.md +++ b/src/scitex/_skills/general/01_arch_04_environment-variables.md @@ -25,6 +25,12 @@ All SciTeX packages MUST use the `SCITEX__*` prefix for environment - Show `$ENV_VAR_NAME` in CLI help defaults, not resolved values - Configuration is external (env vars, config files) — never hardcode secrets or defaults that should be user-configurable +## Resolution precedence + +Per-field precedence is **direct (CLI flag) → config_dict (YAML) → env var → default**. Env vars sit between YAML and built-in defaults — they're a fallback when no YAML config is loaded, never an override of explicit YAML or CLI values. + +Use `scitex_config.PriorityConfig.resolve()` rather than hand-rolling the cascade. See `03_interface_02_cli.md` §6c for the full CLI rule and `~/proj/scitex-config/src/scitex_config/_PriorityConfig.py` for the canonical implementation. + ## Feature Flags All SciTeX feature flags follow the **opt-out** pattern (default enabled, explicitly disable): From 62db8ed5d630dac32b19ff812a3dc87996b5f211 Mon Sep 17 00:00:00 2001 From: Yusuke Watanabe Date: Tue, 28 Apr 2026 11:15:19 +0900 Subject: [PATCH 06/10] docs(skills): add pre-publish wheel verification for package-data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Augments 06_skills_04_editable-installation.md with a concrete pre-tag verification step that catches the silent setuptools failure mode where SKILL.md is on git but absent from the wheel. Real instance: scitex-hpc 0.6.1 (2026-04-28) — package-data entry was missing, build succeeded, CI green, but the wheel didn't contain the SKILL.md. Caught at the unzip-l step before tagging; shipped 0.6.2 the same day with the fix. Adds: why setuptools' packages.find doesn't auto-include markdown data, the 5-second 'unzip -l' pre-tag check with expected output shape, and the post-install belt-and-suspenders verification. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../06_skills_04_editable-installation.md | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/src/scitex/_skills/general/06_skills_04_editable-installation.md b/src/scitex/_skills/general/06_skills_04_editable-installation.md index a981ba91..4bdf730d 100644 --- a/src/scitex/_skills/general/06_skills_04_editable-installation.md +++ b/src/scitex/_skills/general/06_skills_04_editable-installation.md @@ -50,7 +50,37 @@ is_editable = direct_url and '"editable": true' in direct_url "src//_skills" = "/_skills" ``` -- After every wheel-publish, verify a fresh `pip install scitex-` into a clean venv still sees the skills: +### Why setuptools needs the explicit `package-data` entry + +`[tool.setuptools.packages.find] where = ["src"]` only picks up **Python packages** (directories containing `__init__.py`). Markdown files in subdirectories like `_skills//SKILL.md` are NOT auto-included. The result is a silent failure mode: + +- `git ls-files` shows `SKILL.md` is tracked ✅ +- The source tree under `src//_skills/` is intact ✅ +- `python -m build --wheel` builds successfully ✅ +- But the resulting wheel does NOT contain the file ❌ + +PyPI users who `pip install ` see no skill page, and skill-discovery agents iterate over an empty `._skills` namespace. The CI workflow won't catch this because nothing imports the markdown file. + +### Pre-publish verification (5-second check that catches the silent failure) + +After every `python -m build`, **before tagging the release**, verify the wheel actually contains the data files you expect: + +```bash +unzip -l dist/--py3-none-any.whl | grep -E '_skills|SKILL\.md' +``` + +Expected output (one line per shipped skill leaf): +``` +6716 2026-04-28 01:58 /_skills//SKILL.md +``` + +No matching lines = the wheel is missing skills. Re-check `pyproject.toml`'s `[tool.setuptools.package-data]` (or the hatch `force-include` block), rebuild, re-verify. **Don't tag until this passes.** + +A real instance of this trap, scitex-hpc 0.6.1 (2026-04-28): the SKILL.md was added to git but the package-data entry was missing. The wheel built without errors, the version bumped fine, CI was green. Caught at the unzip-l step before the tag was pushed; shipped 0.6.2 with the fix on the same day. Without the unzip check, 0.6.1 would have been an "everything looks done" release that failed silently for every PyPI user. + +### Post-install verification (after the wheel is live) + +For belt-and-suspenders, confirm a fresh `pip install scitex-` into a clean venv resolves the skill: ```bash python -c "from importlib.resources import files; print(list(files('._skills').iterdir()))" From 0afb07af1b673b3176ada8631e8687f063c04f81 Mon Sep 17 00:00:00 2001 From: Yusuke Watanabe Date: Tue, 28 Apr 2026 12:20:25 +0900 Subject: [PATCH 07/10] docs(skills): index two missing leaves + add three failure-playbook entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two general/* leaves existed on disk but weren't linked from SKILL.md (skill-discovery agents couldn't find them): - 04_docs_03_rtd.md (Read the Docs onboarding) - 99_quality_03_packaging-bar.md (packaging quality bar) 98_quality_01_failure-playbook.md gains three entries from concrete incidents during the 2026-04-27/28 multi-tenant scitex-hpc + sac rollout, all documented with symptom / root cause / fix / where-found: §8 a2a-sdk + protobuf 6.x — FieldDescriptor.label AttributeError (caught CI red on sac develop; fix: protobuf<6, not <7) §9 SLURM cgroup kills tmux spawned by srun --overlap (caught Phase 4 prototype; fix: tmux as PID 1 of sbatch script) §10 Chatty login-shell banners break SLURM-output parsing (caught book() polling forever on Spartan; fix: parse line-by-line against a known SLURM-state vocabulary) Future agents hitting any of these symptoms now have the playbook entry to point them straight at the fix. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../general/98_quality_01_failure-playbook.md | 75 +++++++++++++++++++ src/scitex/_skills/general/SKILL.md | 4 +- 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/src/scitex/_skills/general/98_quality_01_failure-playbook.md b/src/scitex/_skills/general/98_quality_01_failure-playbook.md index 6792101e..c6ae9d39 100644 --- a/src/scitex/_skills/general/98_quality_01_failure-playbook.md +++ b/src/scitex/_skills/general/98_quality_01_failure-playbook.md @@ -36,6 +36,7 @@ Cookbook of the specific symptoms observed during ecosystem-wide remediation pas | **LOW** | Skill quality `§2.prefix: MANIFEST.md filename must match NN_kebab-name.md` | MANIFEST.md is a system file, not a leaf | upgrade scitex-dev to a version where the checker exempts `SYSTEM_FILES = {"MANIFEST.md"}` | | **LOW** | Skill quality `§3.index-monolith: SKILL.md > 4096B` | bloated frontmatter description | trim `description:` — it gets copied into skill-matching prompts; verbose prose costs tokens without helping trigger rates | | **LOW** | Skill quality `§4.monolith: NN_foo.md > 10240B` | leaf grew unmanageably | split into two leaves with new prefixes, link both from `SKILL.md`, prefer topical split over length-based | +| **HIGH** | "I added the change and the deploy log shows it landed" — yet the user-visible behavior is unchanged | **Confused "I made the change" with "the change took effect."** Between input and output sit silent defeats: CDN/browser cache, wrong build artifact, wrong code path / wrong host, identity mismatch (same name in two places), unmet preconditions — none throw errors | Verify the EFFECT, not the change. Read the live response / measurement on the deployed system, not the source diff. Quote numbers in the report. (Cross-cuts every package; CSS-specific instances live in package-private skills.) | ## 2. Triage order @@ -149,4 +150,78 @@ for b in bridges: The workflow's `on: push: paths:` filter also excludes `pyproject.toml` — force a run with `gh workflow run "Doc-Drift Nightly" --ref develop` after the push. +## 8. a2a-sdk + protobuf 6.x — `FieldDescriptor.label` AttributeError + +**Symptom (CI):** test runs against `a2a-sdk[http-server]>=1.0.2` fail with:: + + AttributeError: 'google._upb._message.FieldDescriptor' object has no attribute 'label' + +**Root cause.** protobuf **6.x** removed `FieldDescriptor.label` from the +upb backend. a2a-sdk 1.0.x's `validate_proto_required_fields` still +references it (see `a2a/utils/proto_utils.py`). protobuf 7.x continues +to lack the attribute. + +**Fix.** In `pyproject.toml`:: + + "protobuf<6", # not <7 — the bug is in 6.x already + +Don't write `protobuf<7`: that allows 6.x which has the bug. + +**Discovered.** scitex-agent-container 2026-04-27 (CI red on every +develop push since the a2a SDK 1.0 PR). Fixed in v0.5.1 of scitex-hpc +and v0.9.1 of sac. scitex-orochi shipped the same fix on 2026-04-28. + +## 9. SLURM cgroup kills tmux spawned by `srun --overlap` + +**Symptom.** `tmux new-session -d` returns rc=0 inside an `srun --jobid +--overlap` invocation, but `tmux ls` 2 seconds later shows no sessions. +The daemon you just started is gone. + +**Root cause.** SLURM kills *all processes in a step's cgroup* when the +step ends. A tmux daemon spawned by `srun --jobid --overlap …` runs in +that step's transient cgroup, not the job's cgroup. When the wrapping +bash process exits, SIGKILL takes the daemon with it. + +**Fix.** Run tmux as **PID 1 of the sbatch script**, before the hold +body. The daemon then lives in the job's main cgroup and survives +across `srun --overlap` invocations. Tenants connect via the same +named socket:: + + # In the sbatch script body (e.g. from Reservation.book(tmux_server="sac")) + tmux -L sac new-session -d -s _root 'sleep infinity' + tail -f /dev/null # holds the allocation + + # Tenants (run via srun --overlap) attach to the same server + tmux -L sac new-session -d -s tenant-a 'claude --flags ...' + +**Discovered.** scitex-agent-container 2026-04-28, Phase 4 multi-tenant +runtime (verified live on spartan-bm005). The `tmux_server` parameter +on `scitex_hpc.Reservation.book()` exists for exactly this reason. + +## 10. Chatty login-shell banners break SLURM-output parsing + +**Symptom.** ``squeue`` / ``scancel`` / ``scontrol`` calls run via +``ssh 'bash -lc "squeue ..."'`` return data that *contains* the +expected output, but parsers fail because the data is preceded by 5-10 +banner lines like:: + + XAUTHORITY: + DISPLAY: 1.2.3.4:0 + DISPLAY_GPU: :42 + RUNNING node-x + +Naive parsing (``stdout.strip().split()[0]``) returns ``"XAUTHORITY:"`` +as the SLURM state, never matches ``"RUNNING"``, and the polling loop +runs to timeout. + +**Fix.** Parse line-by-line and filter against a known vocabulary +(SLURM states, jobid integers, etc.) — don't trust the first whitespace +token. ``scitex_hpc._reservation._parse_squeue_state_node`` is the +reference implementation. + +**Discovered.** scitex-agent-container 2026-04-28 — first call to +``Reservation.book()`` on Spartan polled forever because Spartan's +``.bashrc`` emits 7 banner lines before any command output. Fixed in +scitex-hpc 0.5.1. + diff --git a/src/scitex/_skills/general/SKILL.md b/src/scitex/_skills/general/SKILL.md index 0fe03615..bf79f42f 100644 --- a/src/scitex/_skills/general/SKILL.md +++ b/src/scitex/_skills/general/SKILL.md @@ -58,6 +58,7 @@ Read in this order when building or auditing a package. Each section presupposes ### 4. Documentation — how does the package become understandable? - [04_docs_01_readme.md](04_docs_01_readme.md) — Standard README template, sections, badges, footer - [04_docs_02_sphinx.md](04_docs_02_sphinx.md) — Sphinx docs, conf.py, RTD config, troubleshooting +- [04_docs_03_rtd.md](04_docs_03_rtd.md) — Read the Docs onboarding, token loading, project import, build triggering ### 5. Version Control — how does it ship? - [05_version-control_01_management.md](05_version-control_01_management.md) — Branches, tags, release waves, release gates (core workflow) @@ -72,8 +73,9 @@ Read in this order when building or auditing a package. Each section presupposes - [06_skills_06_frontmatter-metadata.md](06_skills_06_frontmatter-metadata.md) — Optional YAML fields: `group`, `invocation`, `context_tokens`, `canonical-location`, `see-also` ### 7. Periodic ecosystem quality — run when something feels off -- [98_quality_01_failure-playbook.md](98_quality_01_failure-playbook.md) — Severity-tagged cookbook of ecosystem failure modes +- [98_quality_01_failure-playbook.md](98_quality_01_failure-playbook.md) — Severity-tagged cookbook of ecosystem failure modes (incl. a2a/protobuf, SLURM cgroup tmux, login-shell banner noise) - [99_quality_02_checklist.md](99_quality_02_checklist.md) — Strategic /speak-and-call runbook with append-only log +- [99_quality_03_packaging-bar.md](99_quality_03_packaging-bar.md) — Packaging quality bar (declared deps, peer-pin minimums, extras, install-test CI, twine-first publishing) ### Scratch - [40_playground.md](40_playground.md) — Scratch notes From 011ecb0053662b151faf69a27f9702a9de4547a0 Mon Sep 17 00:00:00 2001 From: Yusuke Watanabe Date: Tue, 28 Apr 2026 12:23:07 +0900 Subject: [PATCH 08/10] docs(skills): refresh release-automation + packaging-bar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 05_version-control_02_release-automation.md: replace deprecated sync-remote / fix-mismatches with the unified ecosystem packages command (--hosts, --packages, --dry-run, --update-hosts) - 99_quality_03_packaging-bar.md: add §5a wheel-vs-source data-file audit (scitex_dev.audit_package_data) catching SKILL.md and other data-file drift before publish --- ...5_version-control_02_release-automation.md | 32 +-- .../general/99_quality_03_packaging-bar.md | 258 ++++++++++++++++++ 2 files changed, 274 insertions(+), 16 deletions(-) create mode 100644 src/scitex/_skills/general/99_quality_03_packaging-bar.md diff --git a/src/scitex/_skills/general/05_version-control_02_release-automation.md b/src/scitex/_skills/general/05_version-control_02_release-automation.md index 4de07706..eb8e9cfc 100644 --- a/src/scitex/_skills/general/05_version-control_02_release-automation.md +++ b/src/scitex/_skills/general/05_version-control_02_release-automation.md @@ -23,7 +23,7 @@ When user says "update all packages" or "full release", for each package: - `mcp__scitex__dev_ecosystem_list` — initial status check across all packages - `mcp__scitex__dev_ecosystem_fix_mismatches` — auto-fix installed vs pyproject.toml mismatches after PyPI publish -- CLI equivalent: `scitex-dev ecosystem fix-mismatches --confirm` +- CLI equivalent: `scitex-dev ecosystem packages --update-hosts` ## Dashboard @@ -49,23 +49,22 @@ scitex-dev show-config # Show resolved dev config scitex-dev search-docs # Search package docs ``` -### Sync +### Sync / verify packages -```bash -scitex-dev ecosystem sync # Local editable reinstall (dry-run default) -scitex-dev ecosystem sync --confirm # Execute -scitex-dev ecosystem sync-remote --host nas # Push to remote host over SSH -scitex-dev ecosystem sync-remote --confirm --host all -``` - -### Fix version mismatches +The unified `ecosystem packages` command observes, previews, or executes +sync against `--reference` (default `origin/develop`): ```bash -scitex-dev ecosystem fix-mismatches # Preview -scitex-dev ecosystem fix-mismatches --confirm # Execute +scitex-dev ecosystem packages # observation: state per (host, package) +scitex-dev ecosystem packages --hosts nas # filter by host (plural noun) +scitex-dev ecosystem packages --packages scitex-io --dry-run # preview the plan +scitex-dev ecosystem packages --hosts all --update-hosts # execute (after dry-run) ``` -Aligns installed version, pyproject toml version, and git tag for every package. +The legacy `sync-remote` and `fix-mismatches` subcommands are deprecated +aliases; they print a redirect and exit 2. + +Aligns installed version, pyproject.toml version, and git tag for every package. ### Utilities @@ -145,11 +144,12 @@ Run `scitex-dev ecosystem list` for the authoritative roster and current version Detects both **version mismatches** (toml != tag != PyPI) and **code-version mismatches** (commits exist since last tag but version not bumped). ```bash -scitex-dev ecosystem fix-mismatches # Preview mismatches -scitex-dev ecosystem fix-mismatches --confirm # Fix them +scitex-dev ecosystem packages # Observation: print mismatches +scitex-dev ecosystem packages --dry-run # Preview the fix plan +scitex-dev ecosystem packages --update-hosts # Execute the fixes ``` -Or via MCP: `mcp__scitex__dev_ecosystem_fix_mismatches`. +Or via MCP: `mcp__scitex__dev_ecosystem_packages`. Python API: diff --git a/src/scitex/_skills/general/99_quality_03_packaging-bar.md b/src/scitex/_skills/general/99_quality_03_packaging-bar.md new file mode 100644 index 00000000..ec25e682 --- /dev/null +++ b/src/scitex/_skills/general/99_quality_03_packaging-bar.md @@ -0,0 +1,258 @@ +--- +name: packaging-quality-bar +description: Ecosystem-wide packaging quality bar for SciTeX packages — declared deps, peer-pin minimums, extras (always all/dev/docs), install-test CI, audit tooling in scitex-dev, twine-first publishing, no bot signatures in commits. Captures the directives that emerged during the 2026-04 standalonization wave. +canonical-location: scitex-python/src/scitex/_skills/general/99_quality_03_packaging-bar.md +tags: [scitex-python, scitex-general, scitex-package, packaging, quality, ci, pypi] +--- + +# SciTeX packaging quality bar + +What every package in the ecosystem must satisfy. Distilled from the +2026-04 standalonization wave (24 new packages + 32 pre-existing audited). +**Every rule below is enforceable via `scitex_dev` and CI** — don't +audit by hand. + +## 1. Don't reinvent the wheel + +Before writing utility code, check PyPI for an existing well-maintained +package. Examples from this ecosystem: + +| Need | Use | +|---|---| +| Title-case strings | `titlecase` (Stuart Colville's NYT-style) | +| TOML parsing | stdlib `tomllib` (3.11+) / `tomli` (3.10) | +| Trove classifier validation | `trove-classifiers` | + +Wrap the dep in a thin domain-specific layer (e.g. SciTeX acronym +callback for `titlecase`). The wrapper goes in our package; the +canonical algorithm stays in the upstream package. + +## 2. Declared dependencies must cover module-level imports + +A package can build, upload, and pass its own tests with the wrong +`pyproject.toml` — the bug only shows up when a fresh user runs +`pip install ` in a clean venv. + +Tools: + +```python +from scitex_dev import audit_dependencies +print(audit_dependencies("/path/to/scitex-foo")) +# → flags missing externals, missing peers, peers without min-version pins +``` + +Rules: + +- Every **module-level non-try-wrapped** external import must be in + `[project.dependencies]` or in an `[project.optional-dependencies]` + extra **and** lazy-imported (try/except in `__init__.py`). +- Heavy deps (`torch`, `tensorflow`, `cv2`) that are only used by a small + fraction of the API go in an extra (`[torch]`, `[cv]`, etc.) and the + imports they trigger are wrapped in `try/except ImportError`. +- Map import → dist names correctly: `cv2` → `opencv-python`, + `bs4` → `beautifulsoup4`, `yaml` → `PyYAML`, `git` → `GitPython`, + `docx` → `python-docx`, `ruamel` → `ruamel.yaml`, + `psycopg2` → `psycopg2-binary`, `umap` → `umap-learn`. + +## 3. SciTeX peer packages must have minimum-version pins + +Always `scitex-X>=Y.Z.W`, never bare `scitex-X`. A bare spec lets the +resolver pick a stale release that lacks a needed feature, breaking +downstream installs silently. + +The audit flags this as `scitex_peers_without_min_version`. + +## 4. `[project.optional-dependencies]` must include `all`, `dev`, `docs` + +```python +from scitex_dev import audit_extras, write_extras_to_pyproject +print(audit_extras("/path/to/scitex-foo")) +# → flags missing all/dev/docs + missing all-refs +write_extras_to_pyproject("/path/to/scitex-foo") # idempotent canonical rewrite +``` + +Conventions: + +- `all` references **every feature extra** (e.g. `pkg[torch]`, `pkg[mcp]`) + so a single `pip install pkg[all]` gets every feature. +- `dev` is `pytest + pytest-cov + ruff` minimum; packages may add custom + dev tools. +- `docs` is `sphinx + sphinx-rtd-theme + myst-parser + sphinx-copybutton + + sphinx-autodoc-typehints` minimum. +- `dev` and `docs` are **intentionally NOT in `all`** (separate concerns: + development setup vs. user feature install). + +## 5. `install-test.yml` CI on every repo + +Every SciTeX repo has `.github/workflows/install-test.yml` that: + +- Builds the wheel +- `pip install dist/*.whl` in a fresh venv on Python 3.10/3.11/3.12 +- Imports the package by detecting the import name from `src/` +- Optionally runs `scitex_dev.audit_dependencies` as a secondary check + +This catches undeclared deps **before publish**, not after a user files +a bug. Template is in `scitex-template/.github/workflows/install-test.yml`. + +## 5a. Wheel-vs-source data-file audit + +Non-Python data files (`SKILL.md`, `*.yaml`, `*.json`, `*.png`, ...) sitting +in `src//` on disk can be silently dropped from the wheel: +`setuptools.packages.find` and hatchling's defaults pick up Python files +plus dirs with `__init__.py` — arbitrary data files are **not shipped** +unless explicitly declared via `[tool.setuptools.package-data]` or +hatchling's `force-include`. + +Failure mode: `pip install scitex-X` works, but `/_skills/SKILL.md` +(or any data leaf) is missing. The package can't load its skill, the +helper can't find its template, etc. + +```python +from scitex_dev import audit_package_data +r = audit_package_data("/path/to/scitex-foo") +print(r.summary()) # one-line pass/fail +if not r.is_clean: + print(r.fix_suggestion) # ready-to-paste pyproject snippet +``` + +Run pre-publish per package; the report includes the exact +pyproject.toml snippet (setuptools or hatchling form, matched to the +package's build-backend) needed to ship the missing files. + +## 6. Twine-first for new ecosystem batches + +PyPI's pending-publisher form is gated to **3 pending entries per +account** and rate-limits the upload endpoint at the **hour scale** for +new project creation. For ecosystems > 3 packages, do this: + +1. `twine upload` once per package (creates the project on PyPI). +2. Attach a trusted publisher from each project's manage page (no + 3-publisher limit there because publishers attach to existing + projects). +3. Future releases use OIDC via `publish-pypi.yml` workflow on `v*` + tag push. + +Reusable helper: + +```python +from scitex_dev import publish, publish_all +publish_all([f"~/proj/{p}" for p in PACKAGES], + method="auto", # 'auto' = OIDC if workflow file exists + dry_run=False, + skip_if_published=True) # idempotent for partial-batch retries +``` + +## 7. Validate trove classifiers locally before build + +PyPI rejects with **400 Bad Request** on upload if any classifier is +unknown. Plausible-looking strings like +`Topic :: Software Development :: Testing :: Benchmark` are *not* in the +trove list. + +```python +from scitex_dev.pypi import validate_classifiers +bad = validate_classifiers("/path/to/scitex-foo") +if bad: + raise ValueError(f"invalid trove classifiers: {bad}") +``` + +`publish_via_twine()` runs this automatically with +`validate_classifiers_first=True`. + +## 8. CLA gates do NOT accept bot-signature commits + +Trailers like `Co-Authored-By: ` from automated tooling +(Claude / Copilot / etc.) cause CLA-assistant to block PR merge with +"unsigned contributor". Drop those trailers — attribution metadata +belongs in PR descriptions, not commit messages. + +The CLA gate matches the **GitHub identity that authored the commit**, +not co-author trailers. + +## 9. Git workflow conventions + +- **Never push directly to `main`.** Use `develop` or feature branches. + The pre-push hook blocks `main` pushes. +- **Always use `git -C /full/path`** in scripts so the CWD doesn't matter. +- **Always use `cp -f`** in scripts to avoid interactive prompts that + hang automation. +- **Always use full paths to `pytest`** in scripts (`pytest --rootdir=/full/path tests/`). + +## 10. Working examples + tests for examples + +Every package should ship: + +- `examples/` directory with a small set of runnable scripts + demonstrating real usage (not docstrings — actual `.py` files). +- `tests/test_examples.py` (or similar) that imports each example or + runs a smoke version of it. + +This catches "the README example actually crashes" bugs at PR time. +**Wave-extracted packages currently lack this** — open follow-up. + +## 11. README + RTD + skills must be uniform-quality + +Each package's README has: + +- 1-line description matching `pyproject.toml` `[project].description` +- Install instructions (`pip install scitex-X`) +- Minimal usage example (real code, copy-paste runnable) +- Status block (alpha/beta/stable + tested-on) +- Badges (PyPI version, Python versions, CI test, Codecov, RTD, License) +- "Part of SciTeX" + Four Freedoms section +- License footer (no `ywatanabe@scitex.ai` per + `02_repo_04_quality.md` — community project) + +RTD must be live for every package (`scitex_dev.check_all_rtd()` audits +this). `_skills/` directory present for any package that exposes +domain-specific patterns to AI agents. + +## 12. Coverage + Codecov badges are part of CI + +Every package's `test.yml` runs `pytest --cov=src/ --cov-report=xml` ++ uploads to Codecov. The README has the resulting badge. **Open +follow-up — currently only ~5/56 packages have this.** + +## 13. Verify before declaring; no false-positive reports + +Always test the actual end-to-end path (e.g. fresh-venv `pip install`) +before saying "fixed." Don't claim work is done based on `python -m +build` succeeding — the real test is `pip install && python -c +"import "` in a clean environment. + +## 14. Skip difficult parts and continue autonomously + +When a particular fix would take too long or requires deep domain +knowledge (e.g. test-quality audits for upstream pre-existing tests), +document it in TODO and move to the next item rather than blocking on +it. The user explicitly prefers progress on multiple fronts over deep +focus on one stuck issue. + +## 15. Ecosystem-wide management lives in `scitex-dev` + +All cross-cutting tooling (audits, publishing, ecosystem registry, +RTD checks, host sync) lives in `scitex_dev`. Don't write one-off +scripts; extend the package. Current modules: + +| Module | Purpose | +|---|---| +| `scitex_dev.ecosystem` | `ECOSYSTEM` dict — single source of truth for package names, paths, repos | +| `scitex_dev.pypi` | `publish` / `publish_all` / `publish_via_tag` / `publish_via_twine` / `trusted_publisher_form` / `validate_classifiers` / `is_published` | +| `scitex_dev._pypi_deps` | `audit_dependencies` / `audit_all` / `DepAuditReport` | +| `scitex_dev._pypi_extras` | `audit_extras` / `write_extras_to_pyproject` | +| `scitex_dev.rtd` | `check_all_rtd` / `check_rtd_status` | +| `scitex_dev.sync` | local ↔ remote host sync | +| `scitex_dev.versions` | version consistency checks | + +When you find a recurring problem, add it as a new audit / fix function +here so the next person doesn't re-discover it. + +## Companion skills + +- `04_development-workflow/12_pypi.md` (global) — full PyPI playbook. +- `04_development-workflow/11_readthedocs.md` (global) — RTD onboarding. +- `04_development-workflow/07_git-versioning.md` (global) — version-bump checklist. +- `02_repo_04_quality.md` — README rules (Four Freedoms, no `as stx`, etc.). +- `01_arch_02_dependency-and-version-pinning.md` — when consumers' minimum versions need bumping. + + From 0103e516ed98ee1c49d68019f47f31259aa3f212 Mon Sep 17 00:00:00 2001 From: Yusuke Watanabe Date: Tue, 28 Apr 2026 13:31:10 +0900 Subject: [PATCH 09/10] fix(deps): declare scitex-config>=0.3.0 (used at runtime) --- pyproject.toml | 1 + .../01_arch_06_local-state-directories.md | 48 ++++++++++++++++++- src/scitex/decorators/_converters.py | 41 +++++++++++++--- src/scitex/decorators/_signal_fn.py | 32 ++++++++++--- 4 files changed, 109 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index be9b42af..b5496bfd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ classifiers = [ # pip install scitex[audio,scholar] # multiple modules # pip install scitex[all] # everything dependencies = [ + "scitex-config>=0.3.0", # Essential Scientific Computing "numpy", "pandas", diff --git a/src/scitex/_skills/general/01_arch_06_local-state-directories.md b/src/scitex/_skills/general/01_arch_06_local-state-directories.md index b3037649..68997c7e 100644 --- a/src/scitex/_skills/general/01_arch_06_local-state-directories.md +++ b/src/scitex/_skills/general/01_arch_06_local-state-directories.md @@ -151,9 +151,55 @@ If a package already ships a different layout (`~/.scitex/_config.yaml`, `~ Do not keep permanent back-compat shims — legacy locations silently defeat `SCITEX_DIR`. -## 9. Related +## 9. Cross-package SoC — each package owns a domain + +The `.scitex//` layout is also the canonical place each package stores config that **other packages then consume**. The rule of thumb: + +> If a question has one obviously-correct answer that should be the same everywhere ("what is this machine called?", "what is the SLURM cluster?", "where is the scholar cache?"), exactly one scitex-* package owns it. Every other package imports its API. + +Anti-pattern: each package re-deriving the answer (e.g. `socket.gethostname()` in five places, getting different results because of FQDN drift, login-node aliasing, container hostnames). When the answer drifts between packages the user sees inconsistency on the dashboard, in logs, in cron entries. + +### Worked example — machine identity (owner: `scitex-resource`) + +`scitex-resource` owns "what machine am I?" because resource detection is its domain. Config lives at `~/.scitex/resource/config.yaml`: + +```yaml +machine: + canonical_name: mba + aliases: + - Yusukes-MacBook-Air + - Yusukes-MacBook-Air.local + role: head + hpc: # optional + cluster: spartan + login_only: true +``` + +API (resolution: `$SCITEX_RESOURCE_MACHINE` → project config → user config → short hostname): + +```python +from scitex_resource import get_machine_name, get_machine_config + +name = get_machine_name() # always returns the same string everywhere +cfg = get_machine_config() # full block with aliases / role / hpc +``` + +Consumers — `scitex-orochi`, `scitex-hpc`, `scitex-agent-container` — call `get_machine_name()` instead of rolling their own hostname logic. The user sets the canonical name once per host; every package agrees. + +### When to make a new package the owner + +You're tempted to add config to `~/.scitex//config.yaml` for a fact that other packages will also need. Ask: + +1. **Is the fact about ``'s domain?** If yes, you own it. Expose a public function. Done. +2. **Is the fact about a domain another scitex-* package already owns?** Consume their API. Don't duplicate the config. +3. **Is the fact ecosystem-wide and no package owns it yet?** Decide who *should* own it (whose name fits best), put the config there, expose the API there. Don't create a "scitex-shared" or "scitex-common" — that's anti-pattern (everyone depends on it, no one feels responsible for it). + +The `runtime/` directory follows the same rule: `/runtime/` is exclusively for *that* package's regenerable state. Never write into another package's `runtime/`. + +## 10. Related - `03_interface_02_cli.md` §6b — config-file resolution uses this layout. - `01_arch_03_modules-and-standalone-packages.md` §5–§6 — `PathManager` dependency-injection pattern. - `01_arch_04_environment-variables.md` — `SCITEX_DIR` and per-package `SCITEX__CONFIG`. - `06_skills_03_public-vs-private.md` — private skills live under `/shared/skills/`. +- `scitex-resource` `_machine.py` — reference implementation of cross-package SoC (machine identity). diff --git a/src/scitex/decorators/_converters.py b/src/scitex/decorators/_converters.py index 21e2565d..a5822bf2 100755 --- a/src/scitex/decorators/_converters.py +++ b/src/scitex/decorators/_converters.py @@ -111,6 +111,28 @@ def to_torch( if device is None: device = kwargs.get("device", "cuda" if torch.cuda.is_available() else "cpu") + def _float_dtype_for(arr: _Any): + """Pick a torch float dtype that preserves the precision of arr. + + - float64 numpy/pandas/xarray inputs -> torch.float64 + - float16 inputs -> torch.float16 + - float32 / integer / unknown -> torch.float32 (current default) + """ + import torch + + np_dtype = getattr(arr, "dtype", None) + if np_dtype is not None: + try: + if np.issubdtype(np_dtype, np.floating): + if np_dtype == np.float64: + return torch.float64 + if np_dtype == np.float16: + return torch.float16 + return torch.float32 + except TypeError: + pass + return torch.float32 + def _to_torch(data: _Any) -> _Any: """Internal conversion function for various data types.""" import pandas as pd @@ -132,8 +154,11 @@ def _to_torch(data: _Any) -> _Any: # Check if it's a numeric array-like structure try: - # Try to convert to tensor directly - new_data = torch.tensor(data).float() + # Promote via numpy first so that mixed-precision inputs follow + # numpy promotion rules (float64 wins over float32, etc.). + np_view = np.asarray(data) + target_dtype = _float_dtype_for(np_view) + new_data = torch.as_tensor(np_view).to(target_dtype) new_data = _try_device(new_data, device) if device == "cuda": _conversion_warning(data, new_data) @@ -155,7 +180,9 @@ def _to_torch(data: _Any) -> _Any: # Handle pandas types if isinstance(data, (pd.Series, pd.DataFrame)): - new_data = torch.tensor(data.to_numpy()).squeeze().float() + np_view = data.to_numpy() + target_dtype = _float_dtype_for(np_view) + new_data = torch.as_tensor(np_view).squeeze().to(target_dtype) new_data = _try_device(new_data, device) if device == "cuda": _conversion_warning(data, new_data) @@ -163,21 +190,23 @@ def _to_torch(data: _Any) -> _Any: # Handle arrays if isinstance(data, np.ndarray): - new_data = torch.tensor(data).float() + target_dtype = _float_dtype_for(data) + new_data = torch.as_tensor(data).to(target_dtype) new_data = _try_device(new_data, device) if device == "cuda": _conversion_warning(data, new_data) return new_data # Handle xarray - import xarray if ( hasattr(data, "__class__") and data.__class__.__module__ == "xarray.core.dataarray" and data.__class__.__name__ == "DataArray" ): - new_data = torch.tensor(np.array(data)).float() + np_view = np.array(data) + target_dtype = _float_dtype_for(np_view) + new_data = torch.as_tensor(np_view).to(target_dtype) new_data = _try_device(new_data, device) if device == "cuda": _conversion_warning(data, new_data) diff --git a/src/scitex/decorators/_signal_fn.py b/src/scitex/decorators/_signal_fn.py index 947debaf..b45d7c2a 100755 --- a/src/scitex/decorators/_signal_fn.py +++ b/src/scitex/decorators/_signal_fn.py @@ -39,6 +39,18 @@ def wrapper(*args: _Any, **kwargs: _Any) -> _Any: # Store original object for type preservation original_object = args[0] if args else None + # Capture the original numpy dtype (if any) so we can restore it on + # the way back. This protects against operations inside ``func`` that + # silently upcast/downcast (e.g. torch ops that promote to float32). + original_dtype = getattr(original_object, "dtype", None) + try: + if original_dtype is not None and not np.issubdtype( + original_dtype, np.floating + ): + original_dtype = None + except TypeError: + original_dtype = None + # Convert only the first argument (signal) to torch tensor if args: # Convert first argument to torch @@ -54,33 +66,41 @@ def wrapper(*args: _Any, **kwargs: _Any) -> _Any: # Convert results back to original input types import torch + def _to_np(t): + arr = t.detach().cpu().numpy() + # Restore original numpy dtype when caller used a real ndarray / + # pandas / xarray with a known floating dtype. + if original_dtype is not None and arr.dtype != original_dtype: + arr = arr.astype(original_dtype, copy=False) + return arr + if isinstance(results, torch.Tensor): if original_object is not None: if isinstance(original_object, list): - return results.detach().cpu().numpy().tolist() + return _to_np(results).tolist() elif isinstance(original_object, np.ndarray): - return results.detach().cpu().numpy() + return _to_np(results) elif ( hasattr(original_object, "__class__") and original_object.__class__.__name__ == "DataFrame" ): import pandas as pd - return pd.DataFrame(results.detach().cpu().numpy()) + return pd.DataFrame(_to_np(results)) elif ( hasattr(original_object, "__class__") and original_object.__class__.__name__ == "Series" ): import pandas as pd - return pd.Series(results.detach().cpu().numpy().flatten()) + return pd.Series(_to_np(results).flatten()) elif ( hasattr(original_object, "__class__") and original_object.__class__.__name__ == "DataArray" ): import xarray as xr - return xr.DataArray(results.detach().cpu().numpy()) + return xr.DataArray(_to_np(results)) return results # Handle tuple returns (e.g., (signal, frequencies)) @@ -93,7 +113,7 @@ def wrapper(*args: _Any, **kwargs: _Any) -> _Any: if original_object is not None and isinstance( original_object, np.ndarray ): - converted_results.append(r.detach().cpu().numpy()) + converted_results.append(_to_np(r)) else: converted_results.append(r) else: From bbc49b96e6663498ce1ed5da0297b2b629a72ae3 Mon Sep 17 00:00:00 2001 From: Yusuke Watanabe Date: Tue, 28 Apr 2026 13:45:02 +0900 Subject: [PATCH 10/10] =?UTF-8?q?chore:=20bump=20version=202.27.3=20?= =?UTF-8?q?=E2=86=92=202.28.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b5496bfd..6d24746d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ build-backend = "hatchling.build" [project] name = "scitex" -version = "2.27.3" +version = "2.28.0" description = "A comprehensive Python library for scientific computing and data analysis" readme = "README.md" requires-python = ">=3.10"