From b74a211fc284cf15b03e0a3cd799d3e9b34b1068 Mon Sep 17 00:00:00 2001 From: Kenil Ajudiya Date: Sat, 20 Dec 2025 02:02:11 +0530 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=90=9B=F0=9F=94=A7=E2=9C=93=20Fixed?= =?UTF-8?q?=20a=20minor=20bug=20in=20downsampling.=20Changed=20default=20o?= =?UTF-8?q?ffset=20to=200.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/xtract2fil/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/xtract2fil/__main__.py b/src/xtract2fil/__main__.py index 8f34c09..c82bc09 100644 --- a/src/xtract2fil/__main__.py +++ b/src/xtract2fil/__main__.py @@ -250,7 +250,7 @@ def xtract2fil( array = array.reshape((-1, int(array.shape[1] // fbin), fbin)).mean(2) array = array.reshape((-1, tbin, array.shape[1])).mean(1) array = array.astype(np.uint8) - array.tofile(filfile_dwnsmp) + array.tofile(filfile) else: # Write the full resolution data only. array.tofile(filfile) @@ -263,7 +263,7 @@ def main( tbin: int = 1, njobs: int = -1, nbeams: int = 10, - offset: int = 64, + offset: int = 0, dual: bool = True, output: str | Path = Path.cwd(), scan: str = "", From 37f63d93300fb4274a1de72577ab9d2c80b4fd4c Mon Sep 17 00:00:00 2001 From: Kenil Ajudiya Date: Sat, 27 Dec 2025 18:02:09 +0530 Subject: [PATCH 2/3] =?UTF-8?q?=E2=9C=9A=20Added=20a=20multi-purpose=20bas?= =?UTF-8?q?h=20script=20wrapper=20for=20SPOTLIGHT=20control=20automation.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- launch_xtract2fil.sh | 281 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 281 insertions(+) create mode 100644 launch_xtract2fil.sh diff --git a/launch_xtract2fil.sh b/launch_xtract2fil.sh new file mode 100644 index 0000000..772104a --- /dev/null +++ b/launch_xtract2fil.sh @@ -0,0 +1,281 @@ +#!/bin/bash + +usage(){ + echo "This script launches xtract2fil on given raw files or directories containing raw files." + echo "The default behaviour is intended for the directory structure of SPOTLIGHT's ParamRudra server." + echo "" + echo "Usage: $0 [OPTIONS] POSITIONAL_ARGS" + echo "" + echo "Options:" + echo " -f FBIN Frequency binning factor (default: 1)" + echo " -t TBIN Time binning factor (default: 1)" + echo " -j NJOBS Number of parallel jobs (default: 16)" + echo " -n NBEAMS Number of beams (default: 10)" + echo " -x OFFSET Offset value (default: 0)" + echo " -d DUAL Dual mode of xtract2fil (true/false, default: true)" + echo " If true, FBIN defaults to 4 and TBIN to 10." + echo " -o OUTPUT_DIR Output directory (default: grand parent of raw files)" + echo " -s SCAN Scan name (default: derived from raw files)" + echo " -h Show help message and exit" + echo "" + echo "Positional arguments:" + echo " FNAME.raw.{0..15} Input raw files" + echo "OR" + echo " OBS_DIR1 [ ... ] Input directories containing raw files" +} + +sanity_checks_for_files(){ + for f in "${file_list[@]}"; do + if [[ ! -e "$f" ]]; then + echo "Warning: file not found: $f" + fi + done +} + +sanity_checks_for_dirs(){ + for DIR in "${DIR_LIST[@]}"; do + if [[ ! -d "$DIR" ]]; then + echo "Warning: directory not found: $DIR" + DIR_LIST=("${DIR_LIST[@]/$DIR}") # remove from list + continue + fi + raw_files=("$DIR"/*.raw.*) + if (( ${#raw_files[@]} == 0 )); then + echo "Warning: no raw files found in directory: $DIR" + DIR_LIST=("${DIR_LIST[@]/$DIR}") # remove from list + continue + fi + done +} + +read_ahdr_file(){ + if [[ -z "$AHDR_FILE" || ! -f "$AHDR_FILE" ]]; then + echo "Error: AHDR file not found: $AHDR_FILE" + return 1 + fi + + # Validate presence of Date and IST Time lines + if ! grep -q '^Date[[:space:]]*=' "$AHDR_FILE"; then + echo "Error: 'Date' line missing in AHDR file: $AHDR_FILE" + echo "Help: Raw files may be corrupted or empty." + return 1 + fi + if ! grep -q '^IST Time[[:space:]]*=' "$AHDR_FILE"; then + echo "Error: 'IST Time' line missing in AHDR file: $AHDR_FILE" + echo "Help: Raw files may be corrupted or empty." + return 1 + fi + + nbeams_per_host=$(sed -n 's/^Total No\. of Beams\/host.*= *//p' "$AHDR_FILE" | head -n1) + if [[ -z "$nbeams_per_host" ]]; then + echo "Error: Failed to parse 'Total No. of Beams/host' from AHDR file: $AHDR_FILE" + return 1 + else + echo "Total No. of Beams/host = $nbeams_per_host" + fi + + if [[ $dual == true ]]; then + dual_flag="--dual" + if [[ $fbin -eq 1 && $tbin -eq 1 ]]; then + # Extract values from AHDR file and determine default fbin and tbin. + channels=$(sed -n 's/^Channels.*= *//p' "$AHDR_FILE" | head -n1) + sampling_time_usec=$(sed -n 's/^Sampling time.*= *//p' "$AHDR_FILE" | head -n1) + + # Basic validations + if [[ -z "$channels" || -z "$sampling_time_usec" ]]; then + echo "Error: Failed to parse required AHDR fields from: $AHDR_FILE" + echo "Channels: '${channels}', Sampling time (uSec): '${sampling_time_usec}'" + return 1 + else + echo "Channels: $channels, Sampling Time (uSec): $sampling_time_usec" + fi + + if (( channels > dwnsmp_channels )); then + fbin=$((dwnsmp_channels / channels)) + else + fbin=1 + fi + if (( $(echo "$sampling_time_usec < $dwnsmp_tsamp" | bc -l) )); then + tbin=$(echo "$dwnsmp_tsamp / $sampling_time_usec" | bc -l) + tbin=${tbin%.*} # floor to integer + else + tbin=1 + fi + fi + else + dual_flag="--no-dual" + fi +} + +xtract_N_chk(){ + echo "Starting xtract2fil for scan: $scan" + echo "Output directory: $output_dir" + echo "Frequency binning factor: $fbin" + echo "Time binning factor: $tbin" + + xtract2fil \ + --fbin "$fbin" \ + --tbin "$tbin" \ + --njobs "$njobs" \ + --nbeams "$nbeams_per_host" \ + --offset "$offset" \ + $dual_flag \ + --output "$output_dir" \ + --scan "$scan" \ + "${file_list[@]}" + + EXIT_CODE=$? + if [[ $EXIT_CODE -ne 0 ]]; then + echo "xtract2fil failed with exit code: $EXIT_CODE" + exit $EXIT_CODE + else + echo "xtract2fil completed successfully" + rm "${file_list[@]}" + + if [[ $dual == true ]]; then + for f in "${ahdr_list[@]}"; do + cp "$f" "$output_dir/FilData/$scan/" + cp "$f" "$output_dir/FilData_dwnsmp/$scan/" + done + else + for f in "${ahdr_list[@]}"; do + cp "$f" "$output_dir" + done + fi + fi +} + +main(){ + # Default parameters + fbin=1 + tbin=1 + njobs=16 + nbeams_per_host=10 + offset=0 + dual=true + output_dir='' + scan='' + inp_list=() + MODE='' # 'files' or 'dirs' + dwnsmp_channels=1024 # default channels after downsampling + dwnsmp_tsamp=13107.2 # default sampling time after downsampling (in microseconds) + + while getopts "f:t:j:n:x:d:o:s:h" opt; do + case $opt in + f) fbin=$OPTARG ;; + t) tbin=$OPTARG ;; + j) njobs=$OPTARG ;; + n) nbeams_per_host=$OPTARG ;; + x) offset=$OPTARG ;; + d) dual=$OPTARG ;; + o) output_dir=$OPTARG ;; + s) scan=$OPTARG ;; + h) usage; exit 0 ;; + *) echo "Invalid option: -$OPTARG"; usage; exit 1 ;; + esac + done + + # Capture positional arguments (filenames) after options + shift $((OPTIND-1)) + if (( $# == 0 )); then + echo "Error: no input files or directories provided as positional arguments" + usage + exit 1 + fi + + inp_list=("$@") + # Expand inp_list to absolute paths + for i in "${!inp_list[@]}"; do + inp_list[$i]=$(realpath "${inp_list[$i]}") + done + + # Determine mode based on the first input argument + if [[ -d "${inp_list[0]}" ]]; then + MODE='dirs' + DIR_LIST=("${inp_list[@]}") + + if [[ -z "$output_dir" ]]; then + DATA_DIR=$(dirname $(dirname "${DIR_LIST[0]}")) + else + DATA_DIR=$output_dir + fi + else + MODE='files' + file_list=("${inp_list[@]}") + + if [[ -z "$scan" ]]; then + scan=$(basename --suffix .raw.0 "${inp_list[0]}") + fi + + if [[ -z "$output_dir" ]]; then + output_dir=$(dirname $(dirname "${file_list[0]}")) + if [[ $dual == false ]]; then + if [[ $fbin -eq 1 && $tbin -eq 1 ]]; then + output_dir="${output_dir}/FilData/$scan" + else + output_dir="${output_dir}/FilData_dwnsmp/$scan" + fi + fi + fi + fi + + sanity_checks_for_${MODE} + + source "/lustre_archive/apps/tdsoft/env.sh" + + if [[ $MODE == 'dirs' ]]; then + for DIR in "${DIR_LIST[@]}"; do + OBS_NAME=$(basename $(dirname "${DIR}")) + + # Determine output directory for this observation. + if [[ $dual == true ]]; then + output_dir="${DATA_DIR}/$OBS_NAME" + else + if [[ $fbin -eq 1 && $tbin -eq 1 ]]; then + output_dir="${DATA_DIR}/${OBS_NAME}/FilData" + else + output_dir="${DATA_DIR}/${OBS_NAME}/FilData_dwnsmp" + fi + fi + + # Start processing each scan in the directory. + for scan in "$DIR"/*.raw.0; do + scan=$(basename --suffix .raw.0 "$scan") + + file_list=("$DIR"/$scan.raw.*) + filtered=() + ahdr_list=() + for f in "${file_list[@]}"; do + if [[ $f == *.raw.*.ahdr ]]; then + ahdr_list+=("$f") + continue + fi + filtered+=("$f") + done + file_list=("${filtered[@]}") + + AHDR_FILE="${ahdr_list[0]}" + read_ahdr_file + if [[ $? -ne 0 ]]; then + echo "Skipping scan $scan due to AHDR file error(s)." + echo "-------------------------------- Invalid data for scan $scan --------------------------------" + continue + fi + + xtract_N_chk + echo "-------------------------------- xtract2fil for scan $scan done --------------------------------" + done + done + else + AHDR_FILE="${file_list[0]}.ahdr" + read_ahdr_file + if [[ $? -ne 0 ]]; then + echo "Cannot process $scan due to AHDR file error(s)." + continue + fi + + xtract_N_chk + fi +} + +main "$@" \ No newline at end of file From 03df0a347e25acb46e66074049c6b03ae3c14783 Mon Sep 17 00:00:00 2001 From: Kenil Ajudiya Date: Sat, 27 Dec 2025 23:46:01 +0530 Subject: [PATCH 3/3] =?UTF-8?q?=E2=9A=A0=EF=B8=8F=20Improved=20error=20han?= =?UTF-8?q?dling=20for=20skipped=20scans.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- launch_xtract2fil.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/launch_xtract2fil.sh b/launch_xtract2fil.sh index 772104a..d3f6006 100644 --- a/launch_xtract2fil.sh +++ b/launch_xtract2fil.sh @@ -224,6 +224,7 @@ main(){ source "/lustre_archive/apps/tdsoft/env.sh" if [[ $MODE == 'dirs' ]]; then + FAILURE_COUNT=0 for DIR in "${DIR_LIST[@]}"; do OBS_NAME=$(basename $(dirname "${DIR}")) @@ -259,6 +260,7 @@ main(){ if [[ $? -ne 0 ]]; then echo "Skipping scan $scan due to AHDR file error(s)." echo "-------------------------------- Invalid data for scan $scan --------------------------------" + FAILURE_COUNT=$((FAILURE_COUNT + 1)) continue fi @@ -266,12 +268,16 @@ main(){ echo "-------------------------------- xtract2fil for scan $scan done --------------------------------" done done + if (( FAILURE_COUNT > 0 )); then + echo "Completed with $FAILURE_COUNT scan(s) skipped due to errors." + return 1 + fi else AHDR_FILE="${file_list[0]}.ahdr" read_ahdr_file if [[ $? -ne 0 ]]; then echo "Cannot process $scan due to AHDR file error(s)." - continue + return 1 fi xtract_N_chk