From e7ebb0c0b3fb75d831d3353f16995e01c6dd1d1f Mon Sep 17 00:00:00 2001 From: Priit Pender Date: Thu, 20 Nov 2025 06:40:48 +0200 Subject: [PATCH 1/4] Add support for WSM mode metadata parsing. Critical feature here is the SRGR polynomial generation from GRSR polynomials for datasets(mainly WSM) with multiple GRSR polynomials in the level 1 ground range data product, this is required for sarsen geocoding interpolation. Datasets with 1 GRSR polynomial(IMP, APP) were left as is for now without the slant range -> ground range index conversion. --- asar_xarray/asar.py | 67 ++++++++++++++++++++++++++++----- asar_xarray/envisat_direct.py | 36 +++++++++++++++--- asar_xarray/records_metadata.py | 36 +++++++++++++++--- 3 files changed, 120 insertions(+), 19 deletions(-) diff --git a/asar_xarray/asar.py b/asar_xarray/asar.py index 0bc1bb5..9b21d2e 100644 --- a/asar_xarray/asar.py +++ b/asar_xarray/asar.py @@ -19,6 +19,46 @@ from asar_xarray.records_metadata import process_records_metadata +def create_srgr_dataset(grsr_poly_coeffs: dict[str, Any], gr_arr: list[float]) -> xr.Dataset: + """ + Build SRGR polynomials from GRSR polynomials of the Envisat file format to mimic + sarsen Sentinel1 GRD handling + + :param grsr_poly_coeffs: grsr polynomial metadata + :param gr_arr: ground range array of the source product + + :return: xarray Datset for srgr polynomial interpolation + """ + + coords: dict[str, Any] = {} + + # TODO may need improvement, see https://github.com/SAR-ARD/asar-xarray/issues/58 + + degree = 10 + coords["degree"] = degree + srgr_coeffs: list[list[float]] = [] + azimuth_time: list[np.datetime64] = [] + + for el in grsr_poly_coeffs: + grsr_poly = np.array(list(reversed(el["grsr_poly_coeffs"]))) + azimuth_time.append(el["azimuth_time"]) + slant_range = np.polyval(grsr_poly, gr_arr) + + srgr_poly = np.polyfit(slant_range[::150], gr_arr[::150], deg=degree) + + # eval_gr = np.polyval(srgr_poly, slant_range) + # diff = gr_arr - eval_gr + # print(diff) + + srgr_coeffs.append(np.array(list(reversed(srgr_poly)))) + + coords["azimuth_time"] = [np.datetime64(dt, "ns") for dt in azimuth_time] + coords["degree"] = list(range(degree + 1)) + data_vars: dict[str, Any] = {"srgrCoefficients": (("azimuth_time", "degree"), srgr_coeffs)} + + return xr.Dataset(data_vars=data_vars, coords=coords) + + def get_metadata(gdal_dataset: gdal.Dataset) -> Dict[str, Any]: """ Build xarray attributes from gdal dataset to be used in xarray. @@ -66,7 +106,7 @@ def open_asar_dataset(filename_or_obj: str | os.PathLike[Any] | ReadBuffer[ if product_str == "Image Mode SLC Image" or product_str == "AP Mode SLC Image": metadata["product_type"] = "SLC" - elif product_str == "Image Mode Precision Image" or product_str == "AP Mode Precision Image": + elif product_str == "Image Mode Precision Image" or product_str == "AP Mode Precision Image" or product_str == "Wide Swath Mode Image": metadata["product_type"] = "GRD" else: raise RuntimeError( @@ -132,7 +172,7 @@ def create_dataset(metadata: dict[str, Any], filepath: str) -> xr.Dataset: if product_str == "Image Mode SLC Image" or product_str == "AP Mode SLC Image": product_type = "SLC" - elif product_str == "Image Mode Precision Image" or product_str == "AP Mode Precision Image": + elif product_str == "Image Mode Precision Image" or product_str == "AP Mode Precision Image" or product_str == "Wide Swath Mode Image": product_type = "GRD" else: raise RuntimeError( @@ -201,16 +241,25 @@ def create_dataset(metadata: dict[str, Any], filepath: str) -> xr.Dataset: number_of_samples, ) - # numpy polyval expects the polynomial top be highest ranked first - coeffs = list(reversed(metadata["direct_parse"]["grsr_coeffs"])) + grsr_arr = metadata["direct_parse"]["grsr_coeffs"] + if len(grsr_arr) == 1: + + # handle 1 GRSR Poly vs N GRSR Poly cases differently, + # numpy polyval expects the polynomial top be highest ranked first - slant_ranges = np.polyval(coeffs, ground_range) - slant_ranges *= 2 + coeffs = list(reversed(grsr_arr[0]["grsr_poly_coeffs"])) - c = 299792458 - slant_range_times = slant_ranges / c + slant_ranges = np.polyval(coeffs, ground_range) + slant_ranges *= 2 - coords["slant_range_time"] = ("pixel", slant_range_times) + c = 299792458 + slant_range_times = slant_ranges / c + + coords["slant_range_time"] = ("pixel", slant_range_times) + else: + swap_dims["pixel"] = "ground_range" + attrs["srgr_conversion"] = create_srgr_dataset(metadata["direct_parse"]["grsr_coeffs"], ground_range) + coords["ground_range"] = ("pixel", ground_range) data = xr.open_dataarray(filepath, engine='rasterio') diff --git a/asar_xarray/envisat_direct.py b/asar_xarray/envisat_direct.py index 69ef382..cbb6115 100644 --- a/asar_xarray/envisat_direct.py +++ b/asar_xarray/envisat_direct.py @@ -5,6 +5,7 @@ import math import pathlib import numpy as np +from asar_xarray import utils from numpy.typing import NDArray @@ -171,6 +172,8 @@ def parse_direct(path: str, gdal_metadata: dict[str, Any], polarization: str) -> factor = math.pow((range_ref / r), spread_loss_power) spreading_loss = np.append(spreading_loss, 1 / factor) + + factor_offset = gdal_metadata["polarization_idx"] cal_factor = gdal_metadata["records"]["main_processing_params"]["calibration_factors"][factor_offset][ "ext_cal_fact"] @@ -286,6 +289,10 @@ def __process_cal_ads(ads: EnvisatADS, gdal_metadata: dict[str, Any], metadata: Tuple containing antenna gains. """ antenna_gains = () + + if gdal_metadata["records"]["main_processing_params"]["ant_elev_corr_flag"]: + return antenna_gains + if ads.name == "EXTERNAL CALIBRATION": aux_folder = pathlib.Path(os.path.abspath(__file__)).parent @@ -342,11 +349,30 @@ def process_sr_gr_ads(ads: EnvisatADS, file_buffer: bytes, metadata: dict[Any, A None """ if ads.name == "SR GR ADS" and ads.size > 0: - srgr_buf = file_buffer[ads.offset:ads.offset + ads.size] - - r = struct.unpack(">ff5f", srgr_buf[13:41]) # Envisat file specification says it is SR/GR Conversion ADSR, # however the polynomial is for GR to SR... - srgr_coeffs = list(r[2:]) - metadata["grsr_coeffs"] = srgr_coeffs + srgr_buf = file_buffer[ads.offset:ads.offset + ads.size] + grsr_coeffs = [] + for i in range(ads.num): + + one_srgr = srgr_buf[i*55:i*55 + 41] + + r = struct.unpack(">iiicff5f", one_srgr) + + srgr_el: dict[str, Any] = {} + + mjd_str = [str(k) for k in r[0:3]] + mjd_str = ",".join(mjd_str) + + srgr_el["azimuth_time"] = utils.get_envisat_time(mjd_str) + srgr_el["slr0"] = r[4] + srgr_el["gr0"] = r[5] + srgr_el["grsr_poly_coeffs"] = r[6:] + grsr_coeffs.append(srgr_el) + + metadata["grsr_coeffs"] = grsr_coeffs + + + + diff --git a/asar_xarray/records_metadata.py b/asar_xarray/records_metadata.py index db26d8f..48e0c39 100644 --- a/asar_xarray/records_metadata.py +++ b/asar_xarray/records_metadata.py @@ -206,8 +206,11 @@ def process_calibration_factors(metadata: dict[str, str]) -> list[dict[str, Any] calib_dict: dict[int, dict[str, Any]] = {} for key, value in metadata.items(): - if not key.startswith('MAIN_PROCESSING_PARAMS_ADS_CALIBRATION_FACTORS'): + + if not "CALIBRATION_FACTORS" in key: continue + #if not key.startswith('MAIN_PROCESSING_PARAMS_ADS_CALIBRATION_FACTORS') or not key.startswith('MAIN_PROCESSING_PARAMS_ADS_0_CALIBRATION_FACTORS'): + # continue # Split key into parts to get index and parameter name parts = key.split('.') @@ -451,6 +454,28 @@ def process_general_main_processing_params(metadata: dict[str, str]) -> dict[str ) processed[new_key] = parsed if parsed is not None else value + # Reuse 0_ tagged metadata for WSM mode + sum_num_output_lines = 0 + keys = processed.keys() + for k in keys: + if "_num_output_lines" in k: + sum_num_output_lines += processed[k] + if "0_range_samp_rate" in keys: + processed["range_samp_rate"] = processed["0_range_samp_rate"] + if "0_range_spacing" in keys: + processed["range_spacing"] = processed["0_range_spacing"] + if "0_radar_freq" in keys: + processed["radar_freq"] = processed["0_radar_freq"] + if "0_azimuth_spacing" in keys: + processed["azimuth_spacing"] = processed["0_azimuth_spacing"] + if "0_ant_elev_corr_flag" in keys: + processed["ant_elev_corr_flag"] = processed["0_ant_elev_corr_flag"] + if sum_num_output_lines > 0: + processed["num_output_lines"] = sum_num_output_lines + + + + return processed @@ -497,6 +522,7 @@ def process_dop_centroid_coeffs(metadata: dict[str, str]) -> dict[str, Any]: """ dop_dict: dict[str, Any] = {} + for key, value in metadata.items(): if not key.startswith('DOP_CENTROID_COEFFS_ADS_'): continue @@ -505,13 +531,13 @@ def process_dop_centroid_coeffs(metadata: dict[str, str]) -> dict[str, Any]: param = key[24:].lower() # Process different parameter types - if param == 'zero_doppler_time': + if 'zero_doppler_time' in param: dop_dict[param] = utils.get_envisat_time(value) - elif param == 'attach_flag': + elif 'attach_flag' in param: dop_dict[param] = bool(int(value)) - elif param == 'dop_conf_below_thresh_flag': + elif 'dop_conf_below_thresh_flag' in param: dop_dict[param] = bool(int(value)) - elif param in ('dop_coef', 'delta_dopp_coeff'): + elif "dop_coef" in param or "delta_dopp_coeff" in param: # Keep all values including zeros for coefficients dop_dict[param] = [float(x) for x in value.strip().split()] else: From ec13e92a490a4dcdb2cd70d969e0984f265e64f8 Mon Sep 17 00:00:00 2001 From: Anton Perepelenko Date: Thu, 4 Dec 2025 13:11:09 +0200 Subject: [PATCH 2/4] Fix typing --- asar_xarray/asar.py | 23 +++++++++++------------ asar_xarray/envisat_direct.py | 4 ++-- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/asar_xarray/asar.py b/asar_xarray/asar.py index 9b21d2e..1299e30 100644 --- a/asar_xarray/asar.py +++ b/asar_xarray/asar.py @@ -1,7 +1,7 @@ """ASAR Xarray Dataset Reader.""" import os -from typing import Dict, Any +from typing import Dict, Any, List import pandas as pd import xarray as xr @@ -19,15 +19,15 @@ from asar_xarray.records_metadata import process_records_metadata -def create_srgr_dataset(grsr_poly_coeffs: dict[str, Any], gr_arr: list[float]) -> xr.Dataset: +def create_srgr_dataset(grsr_poly_coeffs: List[Dict[str, Any]], gr_arr: NDArray[Any]) -> xr.Dataset: """ Build SRGR polynomials from GRSR polynomials of the Envisat file format to mimic sarsen Sentinel1 GRD handling - :param grsr_poly_coeffs: grsr polynomial metadata + :param grsr_poly_coeffs: grsr polynomial metadata (List of dicts) :param gr_arr: ground range array of the source product - :return: xarray Datset for srgr polynomial interpolation + :return: xarray Dataset for srgr polynomial interpolation """ coords: dict[str, Any] = {} @@ -36,24 +36,23 @@ def create_srgr_dataset(grsr_poly_coeffs: dict[str, Any], gr_arr: list[float]) - degree = 10 coords["degree"] = degree - srgr_coeffs: list[list[float]] = [] - azimuth_time: list[np.datetime64] = [] + + srgr_coeffs: List[np.ndarray] = [] + + azimuth_time_raw: List[Any] = [] for el in grsr_poly_coeffs: grsr_poly = np.array(list(reversed(el["grsr_poly_coeffs"]))) - azimuth_time.append(el["azimuth_time"]) + azimuth_time_raw.append(el["azimuth_time"]) slant_range = np.polyval(grsr_poly, gr_arr) srgr_poly = np.polyfit(slant_range[::150], gr_arr[::150], deg=degree) - # eval_gr = np.polyval(srgr_poly, slant_range) - # diff = gr_arr - eval_gr - # print(diff) - srgr_coeffs.append(np.array(list(reversed(srgr_poly)))) - coords["azimuth_time"] = [np.datetime64(dt, "ns") for dt in azimuth_time] + coords["azimuth_time"] = [np.datetime64(dt, "ns") for dt in azimuth_time_raw] coords["degree"] = list(range(degree + 1)) + data_vars: dict[str, Any] = {"srgrCoefficients": (("azimuth_time", "degree"), srgr_coeffs)} return xr.Dataset(data_vars=data_vars, coords=coords) diff --git a/asar_xarray/envisat_direct.py b/asar_xarray/envisat_direct.py index cbb6115..035e854 100644 --- a/asar_xarray/envisat_direct.py +++ b/asar_xarray/envisat_direct.py @@ -362,8 +362,8 @@ def process_sr_gr_ads(ads: EnvisatADS, file_buffer: bytes, metadata: dict[Any, A srgr_el: dict[str, Any] = {} - mjd_str = [str(k) for k in r[0:3]] - mjd_str = ",".join(mjd_str) + mjd_arr = [str(k) for k in r[0:3]] + mjd_str = ",".join(mjd_arr) srgr_el["azimuth_time"] = utils.get_envisat_time(mjd_str) srgr_el["slr0"] = r[4] From 71e8e640ceacc9aa70663a2add6d6effcb0e8a75 Mon Sep 17 00:00:00 2001 From: Anton Perepelenko Date: Thu, 4 Dec 2025 13:14:41 +0200 Subject: [PATCH 3/4] Fix formatting --- asar_xarray/asar.py | 6 ++++-- asar_xarray/envisat_direct.py | 9 +-------- asar_xarray/records_metadata.py | 9 +++------ 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/asar_xarray/asar.py b/asar_xarray/asar.py index 1299e30..443cc41 100644 --- a/asar_xarray/asar.py +++ b/asar_xarray/asar.py @@ -105,7 +105,8 @@ def open_asar_dataset(filename_or_obj: str | os.PathLike[Any] | ReadBuffer[ if product_str == "Image Mode SLC Image" or product_str == "AP Mode SLC Image": metadata["product_type"] = "SLC" - elif product_str == "Image Mode Precision Image" or product_str == "AP Mode Precision Image" or product_str == "Wide Swath Mode Image": + elif (product_str == "Image Mode Precision Image" or product_str == "AP Mode Precision Image" + or product_str == "Wide Swath Mode Image"): metadata["product_type"] = "GRD" else: raise RuntimeError( @@ -171,7 +172,8 @@ def create_dataset(metadata: dict[str, Any], filepath: str) -> xr.Dataset: if product_str == "Image Mode SLC Image" or product_str == "AP Mode SLC Image": product_type = "SLC" - elif product_str == "Image Mode Precision Image" or product_str == "AP Mode Precision Image" or product_str == "Wide Swath Mode Image": + elif (product_str == "Image Mode Precision Image" or product_str == "AP Mode Precision Image" + or product_str == "Wide Swath Mode Image"): product_type = "GRD" else: raise RuntimeError( diff --git a/asar_xarray/envisat_direct.py b/asar_xarray/envisat_direct.py index 035e854..4d6fb0e 100644 --- a/asar_xarray/envisat_direct.py +++ b/asar_xarray/envisat_direct.py @@ -172,8 +172,6 @@ def parse_direct(path: str, gdal_metadata: dict[str, Any], polarization: str) -> factor = math.pow((range_ref / r), spread_loss_power) spreading_loss = np.append(spreading_loss, 1 / factor) - - factor_offset = gdal_metadata["polarization_idx"] cal_factor = gdal_metadata["records"]["main_processing_params"]["calibration_factors"][factor_offset][ "ext_cal_fact"] @@ -355,8 +353,7 @@ def process_sr_gr_ads(ads: EnvisatADS, file_buffer: bytes, metadata: dict[Any, A srgr_buf = file_buffer[ads.offset:ads.offset + ads.size] grsr_coeffs = [] for i in range(ads.num): - - one_srgr = srgr_buf[i*55:i*55 + 41] + one_srgr = srgr_buf[i * 55:i * 55 + 41] r = struct.unpack(">iiicff5f", one_srgr) @@ -372,7 +369,3 @@ def process_sr_gr_ads(ads: EnvisatADS, file_buffer: bytes, metadata: dict[Any, A grsr_coeffs.append(srgr_el) metadata["grsr_coeffs"] = grsr_coeffs - - - - diff --git a/asar_xarray/records_metadata.py b/asar_xarray/records_metadata.py index 48e0c39..5e7bece 100644 --- a/asar_xarray/records_metadata.py +++ b/asar_xarray/records_metadata.py @@ -207,9 +207,10 @@ def process_calibration_factors(metadata: dict[str, str]) -> list[dict[str, Any] for key, value in metadata.items(): - if not "CALIBRATION_FACTORS" in key: + if "CALIBRATION_FACTORS" not in key: continue - #if not key.startswith('MAIN_PROCESSING_PARAMS_ADS_CALIBRATION_FACTORS') or not key.startswith('MAIN_PROCESSING_PARAMS_ADS_0_CALIBRATION_FACTORS'): + # if not key.startswith('MAIN_PROCESSING_PARAMS_ADS_CALIBRATION_FACTORS') + # or not key.startswith('MAIN_PROCESSING_PARAMS_ADS_0_CALIBRATION_FACTORS'): # continue # Split key into parts to get index and parameter name @@ -473,9 +474,6 @@ def process_general_main_processing_params(metadata: dict[str, str]) -> dict[str if sum_num_output_lines > 0: processed["num_output_lines"] = sum_num_output_lines - - - return processed @@ -522,7 +520,6 @@ def process_dop_centroid_coeffs(metadata: dict[str, str]) -> dict[str, Any]: """ dop_dict: dict[str, Any] = {} - for key, value in metadata.items(): if not key.startswith('DOP_CENTROID_COEFFS_ADS_'): continue From 480e8de18d86ecac456796e8d59db13ee3744acf Mon Sep 17 00:00:00 2001 From: Anton Perepelenko Date: Thu, 4 Dec 2025 13:20:22 +0200 Subject: [PATCH 4/4] Fix formatting --- asar_xarray/asar.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/asar_xarray/asar.py b/asar_xarray/asar.py index 443cc41..fab08f3 100644 --- a/asar_xarray/asar.py +++ b/asar_xarray/asar.py @@ -21,15 +21,13 @@ def create_srgr_dataset(grsr_poly_coeffs: List[Dict[str, Any]], gr_arr: NDArray[Any]) -> xr.Dataset: """ - Build SRGR polynomials from GRSR polynomials of the Envisat file format to mimic - sarsen Sentinel1 GRD handling + Build SRGR polynomials from GRSR polynomials of the Envisat file format to mimic sarsen Sentinel1 GRD handling. :param grsr_poly_coeffs: grsr polynomial metadata (List of dicts) :param gr_arr: ground range array of the source product :return: xarray Dataset for srgr polynomial interpolation """ - coords: dict[str, Any] = {} # TODO may need improvement, see https://github.com/SAR-ARD/asar-xarray/issues/58 @@ -37,7 +35,7 @@ def create_srgr_dataset(grsr_poly_coeffs: List[Dict[str, Any]], gr_arr: NDArray[ degree = 10 coords["degree"] = degree - srgr_coeffs: List[np.ndarray] = [] + srgr_coeffs: List[NDArray[Any]] = [] azimuth_time_raw: List[Any] = []