diff --git a/Docker/Dockerfile b/Docker/Dockerfile index 5e5a4edf..bb4242e3 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -2,7 +2,7 @@ FROM oraclelinux:8 LABEL author="OPERA ADT" \ description="RTC cal/val release R4" \ - version="1.0.3-final" + version="1.0.4-final" RUN yum -y update &&\ yum -y install curl &&\ diff --git a/Docker/requirements.txt b/Docker/requirements.txt index 2bc9ee0f..4d7ec4d2 100644 --- a/Docker/requirements.txt +++ b/Docker/requirements.txt @@ -1,4 +1,4 @@ -python>=3.9,<3.10 +python cmake>=3.18 eigen>=3.3 fftw>=3.3 @@ -16,9 +16,12 @@ pyre>=1.11.2 pytest cython ruamel.yaml -scipy!=1.10.0 +scipy setuptools shapely yamale backoff -isce3==0.15.0 \ No newline at end of file +isce3 +libnetcdf +libgdal-hdf5 +libgdal-netcdf diff --git a/build_docker_image.sh b/build_docker_image.sh index 5f749d64..aee9a3fe 100755 --- a/build_docker_image.sh +++ b/build_docker_image.sh @@ -2,7 +2,7 @@ REPO=opera IMAGE=rtc -TAG=final_1.0.3 +TAG=final_1.0.4 echo "IMAGE is $REPO/$IMAGE:$TAG" diff --git a/src/rtc/defaults/rtc_s1.yaml b/src/rtc/defaults/rtc_s1.yaml index fd3e2a6b..6a4fb8f7 100644 --- a/src/rtc/defaults/rtc_s1.yaml +++ b/src/rtc/defaults/rtc_s1.yaml @@ -197,9 +197,6 @@ runconfig: # OPTIONAL - Choices: "single_block", "geogrid", "geogrid_and_radargrid", and "auto" (default) memory_mode: - # OPTIONAL - Processing upsampling factor applied to input geogrid - geogrid_upsampling: 1 - # Save the incidence angle save_incidence_angle: False diff --git a/src/rtc/defaults/rtc_s1_static.yaml b/src/rtc/defaults/rtc_s1_static.yaml index 001b44cf..f917ca8d 100644 --- a/src/rtc/defaults/rtc_s1_static.yaml +++ b/src/rtc/defaults/rtc_s1_static.yaml @@ -198,9 +198,6 @@ runconfig: # OPTIONAL - Choices: "single_block", "geogrid", "geogrid_and_radargrid", and "auto" (default) memory_mode: - # OPTIONAL - Processing upsampling factor applied to input geogrid - geogrid_upsampling: 1 - # Save the incidence angle save_incidence_angle: True diff --git a/src/rtc/h5_prep.py b/src/rtc/h5_prep.py index 61fdb6fb..26636d03 100644 --- a/src/rtc/h5_prep.py +++ b/src/rtc/h5_prep.py @@ -181,7 +181,7 @@ def save_hdf5_file(hdf5_obj, output_hdf5_file, clip_max, del hdf5_obj[h5_ds] pol_list_s2 = np.array(pol_list, dtype='S2') dset = hdf5_obj.create_dataset(h5_ds, data=pol_list_s2) - dset.attrs['description'] = np.string_( + dset.attrs['description'] = np.bytes_( 'List of processed polarization layers') # save geogrid coordinates @@ -284,11 +284,11 @@ def create_hdf5_file(product_id, output_hdf5_file, orbit, burst, cfg, ''' hdf5_obj = h5py.File(output_hdf5_file, 'w') - hdf5_obj.attrs['Conventions'] = np.string_("CF-1.8") - hdf5_obj.attrs["contact"] = np.string_("operasds@jpl.nasa.gov") - hdf5_obj.attrs["institution"] = np.string_("NASA JPL") - hdf5_obj.attrs["project"] = np.string_("OPERA") - hdf5_obj.attrs["reference_document"] = np.string_( + hdf5_obj.attrs['Conventions'] = np.bytes_("CF-1.8") + hdf5_obj.attrs["contact"] = np.bytes_("operasds@jpl.nasa.gov") + hdf5_obj.attrs["institution"] = np.bytes_("NASA JPL") + hdf5_obj.attrs["project"] = np.bytes_("OPERA") + hdf5_obj.attrs["reference_document"] = np.bytes_( "Product Specification Document for the OPERA Radiometric" " Terrain-Corrected SAR Backscatter from Sentinel-1," " JPL D-108758, Rev. Working Version 1, Aug 31, 2023") @@ -296,9 +296,9 @@ def create_hdf5_file(product_id, output_hdf5_file, orbit, burst, cfg, # product type product_type = cfg.groups.primary_executable.product_type if product_type == STATIC_LAYERS_PRODUCT_TYPE: - hdf5_obj.attrs["title"] = np.string_("OPERA RTC-S1-STATIC Product") + hdf5_obj.attrs["title"] = np.bytes_("OPERA RTC-S1-STATIC Product") else: - hdf5_obj.attrs["title"] = np.string_("OPERA RTC-S1 Product") + hdf5_obj.attrs["title"] = np.bytes_("OPERA RTC-S1 Product") populate_metadata_group(product_id, hdf5_obj, burst, cfg, processing_datetime, is_mosaic) @@ -310,23 +310,44 @@ def create_hdf5_file(product_id, output_hdf5_file, orbit, burst, cfg, def save_orbit(orbit, orbit_group, orbit_file_path): + + # ensure that the orbit reference epoch has not fractional part + # otherwise, trancate it to seconds precision + orbit_reference_epoch = orbit.reference_epoch + if orbit_reference_epoch.frac != 0: + logger.warning('the orbit reference epoch is not an' + ' integer number. Truncating it' + ' to seconds precision and' + ' updating the orbit ephemeris' + ' accordingly.') + + epoch = isce3.core.DateTime(orbit_reference_epoch.year, + orbit_reference_epoch.month, + orbit_reference_epoch.day, + orbit_reference_epoch.hour, + orbit_reference_epoch.minute, + orbit_reference_epoch.second) + + orbit.update_reference_epoch(epoch) + orbit.save_to_h5(orbit_group) + # Add description attributes. - orbit_group["time"].attrs["description"] = np.string_( + orbit_group["time"].attrs["description"] = np.bytes_( "Time vector record. This" " record contains the time corresponding to position, velocity," " acceleration records") - orbit_group["position"].attrs["description"] = np.string_( + orbit_group["position"].attrs["description"] = np.bytes_( "Position vector" " record. This record contains the platform position data with" " respect to WGS84 G1762 reference frame") - orbit_group["velocity"].attrs["description"] = np.string_( + orbit_group["velocity"].attrs["description"] = np.bytes_( "Velocity vector" " record. This record contains the platform velocity data with" " respect to WGS84 G1762 reference frame") orbit_group.create_dataset( 'referenceEpoch', - data=np.string_(orbit.reference_epoch.isoformat())) + data=np.bytes_(orbit.reference_epoch.isoformat())) # Orbit source/type orbit_type = 'Undefined' @@ -349,9 +370,12 @@ def save_orbit(orbit, orbit_group, orbit_file_path): orbit_type_list.append(orbit_type_individual) orbit_type = '; '.join(orbit_type_list) - d = orbit_group.require_dataset("orbitType", (), "S64", - data=np.string_(orbit_type)) - d.attrs["description"] = np.string_( + if 'orbitType' in orbit_group: + del orbit_group['orbitType'] + d = orbit_group.create_dataset("orbitType", + data=np.bytes_(orbit_type)) + + d.attrs["description"] = np.bytes_( "Type of orbit file used in processing") @@ -1286,11 +1310,11 @@ def populate_metadata_group(product_id: str, continue if isinstance(data, str): dset = h5py_obj.create_dataset( - path_dataset_in_h5, data=np.string_(data)) + path_dataset_in_h5, data=np.bytes_(data)) else: dset = h5py_obj.create_dataset(path_dataset_in_h5, data=data) - dset.attrs['description'] = np.string_(description) + dset.attrs['description'] = np.bytes_(description) def save_hdf5_dataset(ds_filename, h5py_obj, root_path, @@ -1331,7 +1355,11 @@ def save_hdf5_dataset(ds_filename, h5py_obj, root_path, logger.warning(f'WARNING Cannot open raster file: {ds_filename}') return - ds_name = layer_hdf5_dict[layer_name] + if isinstance(layer_name, str): + ds_name = layer_hdf5_dict[layer_name] + else: + ds_name = [layer_hdf5_dict[l] for l in layer_name] + if long_name is not None: description = long_name else: @@ -1367,18 +1395,18 @@ def save_hdf5_dataset(ds_filename, h5py_obj, root_path, dset = h5py_obj.create_dataset(h5_ds, data=data) dset.dims[0].attach_scale(yds) dset.dims[1].attach_scale(xds) - dset.attrs['grid_mapping'] = np.string_("projection") + dset.attrs['grid_mapping'] = np.bytes_("projection") if standard_name is not None: - dset.attrs['standard_name'] = np.string_(standard_name) + dset.attrs['standard_name'] = np.bytes_(standard_name) if long_name is not None: - dset.attrs['long_name'] = np.string_(long_name) + dset.attrs['long_name'] = np.bytes_(long_name) - dset.attrs['description'] = np.string_(description) + dset.attrs['description'] = np.bytes_(description) if units is not None: - dset.attrs['units'] = np.string_(units) + dset.attrs['units'] = np.bytes_(units) if fill_value is not None: dset.attrs.create('_FillValue', data=fill_value) @@ -1386,6 +1414,8 @@ def save_hdf5_dataset(ds_filename, h5py_obj, root_path, dset.attrs.create('_FillValue', data=np.nan + 1j * np.nan) elif 'float' in gdal.GetDataTypeName(raster.datatype()).lower(): dset.attrs.create('_FillValue', data=np.nan) + elif 'byte' in gdal.GetDataTypeName(raster.datatype()).lower(): + dset.attrs.create('_FillValue', data=255) if stats_vector is not None: stats_obj = stats_vector[band] diff --git a/src/rtc/rtc_s1.py b/src/rtc/rtc_s1.py index 1714146e..14b908fe 100755 --- a/src/rtc/rtc_s1.py +++ b/src/rtc/rtc_s1.py @@ -769,22 +769,21 @@ def run_parallel(cfg: RunConfig, logfile_path, flag_logger_full_format): if flag_layover_shadow_mask_is_temporary: temp_files_list.append(layover_shadow_mask_file) - layover_shadow_mask_file = None else: burst_output_file_list.append(layover_shadow_mask_file) logger.info(f'file saved: {layover_shadow_mask_file}') - # Take the layover shadow mask from HDF5 file if not exists - if save_secondary_layers_as_hdf5: - layover_shadow_mask_file = ( - f'NETCDF:{burst_hdf5_in_output}:' - f'{DATA_BASE_GROUP}/' - f'{layer_hdf5_dict[LAYER_NAME_LAYOVER_SHADOW_MASK]}') - - if save_mask: - output_metadata_dict[ - LAYER_NAME_LAYOVER_SHADOW_MASK][1].append( - layover_shadow_mask_file) + # Take the layover shadow mask from HDF5 file if not exists + if save_secondary_layers_as_hdf5: + layover_shadow_mask_file = ( + f'NETCDF:{burst_hdf5_in_output}:' + f'{DATA_BASE_GROUP}/' + f'{layer_hdf5_dict[LAYER_NAME_LAYOVER_SHADOW_MASK]}') + + if save_mask: + output_metadata_dict[ + LAYER_NAME_LAYOVER_SHADOW_MASK][1].append( + layover_shadow_mask_file) if not save_mask: layover_shadow_mask_file = None diff --git a/src/rtc/rtc_s1_single_job.py b/src/rtc/rtc_s1_single_job.py index 098514f1..60af24c7 100755 --- a/src/rtc/rtc_s1_single_job.py +++ b/src/rtc/rtc_s1_single_job.py @@ -1214,7 +1214,6 @@ def run_single_job(cfg: RunConfig): cfg.groups.processing.correction_lut_range_spacing_in_meters memory_mode = geocode_namespace.memory_mode - geogrid_upsampling = geocode_namespace.geogrid_upsampling shadow_dilation_size = geocode_namespace.shadow_dilation_size abs_cal_factor = geocode_namespace.abs_rad_cal clip_max = geocode_namespace.clip_max @@ -1323,7 +1322,7 @@ def run_single_job(cfg: RunConfig): logger.info('Save layers:') logger.info(f' {layer_names_dict[LAYER_NAME_LAYOVER_SHADOW_MASK]}:' - f' {save_rtc_anf}') + f' {save_mask}') logger.info(f' RTC area normalization factor: {save_rtc_anf}') logger.info(f' RTC area normalization factor Gamma0 to Beta0:' f' {save_rtc_anf_gamma0_to_sigma0}') @@ -1719,9 +1718,10 @@ def run_single_job(cfg: RunConfig): else: burst_output_file_list.append(layover_shadow_mask_file) logger.info(f'file saved: {layover_shadow_mask_file}') - if save_mask: - output_metadata_dict[LAYER_NAME_LAYOVER_SHADOW_MASK][1].append( - layover_shadow_mask_file) + + if save_mask: + output_metadata_dict[LAYER_NAME_LAYOVER_SHADOW_MASK][1].append( + layover_shadow_mask_file) if not save_mask: layover_shadow_mask_file = None @@ -1803,6 +1803,8 @@ def run_single_job(cfg: RunConfig): geo_obj.geogrid(geogrid.start_x, geogrid.start_y, geogrid.spacing_x, geogrid.spacing_y, geogrid.width, geogrid.length, geogrid.epsg) + + geogrid_upsampling = 1 geo_obj.geocode(radar_grid=radar_grid, input_raster=rdr_burst_raster, @@ -1848,8 +1850,7 @@ def run_single_job(cfg: RunConfig): if product_type != STATIC_LAYERS_PRODUCT_TYPE: output_imagery_list.append(geo_burst_filename) - if (flag_process and save_mask and - not save_secondary_layers_as_hdf5): + if flag_process and save_mask: set_mask_fill_value_and_ctable(layover_shadow_mask_file, geo_burst_filename) diff --git a/src/rtc/runconfig.py b/src/rtc/runconfig.py index c2a6b67e..507b09fc 100644 --- a/src/rtc/runconfig.py +++ b/src/rtc/runconfig.py @@ -39,9 +39,6 @@ def load_parameters(cfg): if geocode_namespace.clip_min is None: geocode_namespace.clip_min = np.nan - if geocode_namespace.geogrid_upsampling is None: - geocode_namespace.geogrid_upsampling = 1.0 - if geocode_namespace.memory_mode == 'single_block': geocode_namespace.memory_mode = \ isce3.core.GeocodeMemoryMode.SingleBlock diff --git a/src/rtc/schemas/rtc_s1.yaml b/src/rtc/schemas/rtc_s1.yaml index 9c2798d4..0ef54563 100644 --- a/src/rtc/schemas/rtc_s1.yaml +++ b/src/rtc/schemas/rtc_s1.yaml @@ -223,9 +223,6 @@ geocoding_options: # Memory mode memory_mode: enum('auto', 'single_block', 'geogrid', 'geogrid_and_radargrid', required=False) - # Processing upsampling factor on top of the input geogrid - geogrid_upsampling: int(required=False) - # Save the incidence angle save_incidence_angle: bool(required=False) diff --git a/src/rtc/version.py b/src/rtc/version.py index 0c2831a5..2f1fca82 100644 --- a/src/rtc/version.py +++ b/src/rtc/version.py @@ -1 +1 @@ -VERSION='1.0.3' +VERSION='1.0.4' diff --git a/tests/runconfigs/s1b_los_angeles_mask_off.yaml b/tests/runconfigs/s1b_los_angeles_mask_off.yaml new file mode 100644 index 00000000..b1aa9cbd --- /dev/null +++ b/tests/runconfigs/s1b_los_angeles_mask_off.yaml @@ -0,0 +1,211 @@ +runconfig: + name: rtc_s1_workflow_default + + groups: + primary_executable: + product_type: RTC_S1 + + pge_name_group: + pge_name: RTC_S1_PGE + + input_file_group: + # Required. List of SAFE files (min=1) + safe_file_path: [data/s1b_los_angeles/input_dir/S1B_IW_SLC__1SDV_20200926T135152_20200926T135219_023547_02CBCC_F988-CROPPED.zip] + + # Required. List of orbit (EOF) files (min=1) + orbit_file_path: [data/s1b_los_angeles/input_dir/S1B_OPER_AUX_PREORB_OPOD_20200926T111139_V20200926T095426_20200926T162926.EOF] + + # Optional. Burst ID to process (empty for all bursts) + # burst_id: [t069_147173_iw1, t069_147174_iw1] + burst_id: [t071_151225_iw1, t071_151226_iw1] + + dynamic_ancillary_file_group: + # Digital elevation model + dem_file: data/s1b_los_angeles/input_dir/dem.tif + + # Digital elevation model description + dem_file_description: + + static_ancillary_file_group: + + # burst database sqlite file + burst_database_file: + + product_group: + processing_type: 'CUSTOM' + + # Directory where PGE will place results + product_path: . + + # Directory where SAS writes temporary data + scratch_path: data/s1b_los_angeles/scratch_dir + + # If option `save_bursts` is set, output bursts are saved to: + # {output_dir}/{burst_id}/{product_id}{suffix}.{ext} + # If option `save_mosaics` is set, output mosaics are saved to: + # {output_dir}/{product_id}{suffix}.{ext} + # If the field `product_id`` is left empty, the prefix "rtc_product" + # will be used instead. + # `suffix` is only used when there are multiple output files. + # `ext` is determined by geocoding_options.output_imagery_format. + output_dir: data/s1b_los_angeles/output_dir + product_id: OPERA_L2_RTC-S1_071-151230-IW2_20200926T135152Z_20230125T134122Z_S1B_30 + + save_bursts: True + save_mosaics: True + output_imagery_format: COG + output_imagery_compression: ZSTD + output_imagery_nbits: 16 + + # Optional. Save secondary layers (e.g., inc. angle) within + # the HDF5 file + save_secondary_layers_as_hdf5: False + + # Save RTC-S1 metadata in the HDF5 format + # Optional for `output_imagery_format` equal to 'ENVI', 'GTiff', or + # 'COG', and enabled by default for `output_imagery_format` equal + # to 'HDF5' or 'NETCDF' or `save_secondary_layers_as_hdf5` is True + save_metadata: True + + processing: + + # Check if ancillary input cover entirely output products + check_ancillary_inputs_coverage: True + + polarization: co-pol + + # Options to run geo2rdr + geo2rdr: + threshold: 1.0e-8 + numiter: 25 + + # Options to run rdr2geo + rdr2geo: + threshold: 1.0e-7 + numiter: 25 + + # Apply absolute radiometric correction + apply_absolute_radiometric_correction: True + + # Apply thermal noise correction + apply_thermal_noise_correction: True + + # OPTIONAL - Apply RTC + apply_rtc: True + + # Apply bistatic delay correction + apply_bistatic_delay_correction: True + + # Apply static tropospheric delay correction + apply_static_tropospheric_delay_correction: True + + # OPTIONAL - to control behavior of RTC module + # (only applicable if geocode.apply_rtc is True) + rtc: + # OPTIONAL - Choices: + # "gamma0" (default) + # "sigma0" + output_type: gamma0 + + # OPTIONAL - Choices: + # "bilinear_distribution" (default) + # "area_projection" + algorithm_type: area_projection + + # OPTIONAL - Choices: + # "beta0" (default) + # "sigma0" + input_terrain_radiometry: beta0 + + # OPTIONAL - Minimum RTC area factor in dB + rtc_min_value_db: -30 + + # RTC DEM upsampling + dem_upsampling: 1 + + # OPTIONAL - Mechanism to specify output posting and DEM + geocoding: + + # OPTIONAL - Apply valid-samples sub-swath masking + apply_valid_samples_sub_swath_masking: True + + # OPTIONAL - Apply shadow masking + apply_shadow_masking: False + + # OPTIONAL - + algorithm_type: area_projection + + # OPTIONAL - Choices: "single_block", "geogrid", "geogrid_radargrid", and "auto" (default) + memory_mode: auto + + # Save the incidence angle + save_incidence_angle: False + + # Save the local-incidence angle + save_local_inc_angle: True + + # Save the projection angle + save_projection_angle: False + + # Save the RTC ANF compuated with the projection angle method + save_rtc_anf_projection_angle: False + + # Save the range slope angle + save_range_slope: False + + # Save the number of looks used to compute RTC-S1 + save_nlooks: True + + # Save the RTC area normalization factor (ANF) used to generate + # the RTC product + save_rtc_anf: True + + # Save the RTC area normalization factor (ANF) gamma0 to sigma0 + save_rtc_anf_gamma0_to_sigma0: False + + # Save interpolated DEM used to compute RTC-S1 + save_dem: False + + # Save layover shadow mask + save_mask: False + + # OPTIONAL - Absolute radiometric correction + abs_rad_cal: 1 + + # OPTIONAL - Clip values above threshold + clip_max: + + # OPTIONAL - Clip values below threshold + clip_min: + + # OPTIONAL - Double sampling of the radar-grid + # input sampling in the range direction + upsample_radargrid: False + + bursts_geogrid: + output_epsg: + x_posting: 300 + y_posting: 300 + x_snap: 300 + y_snap: 300 + top_left: + x: + y: + bottom_right: + x: + y: + + + mosaicking: + mosaic_geogrid: + output_epsg: + x_posting: 300 + y_posting: 300 + x_snap: 300 + y_snap: 300 + top_left: + x: + y: + bottom_right: + x: + y: diff --git a/tests/runconfigs/s1b_los_angeles.yaml b/tests/runconfigs/s1b_los_angeles_mask_off_h5.yaml similarity index 96% rename from tests/runconfigs/s1b_los_angeles.yaml rename to tests/runconfigs/s1b_los_angeles_mask_off_h5.yaml index 4f230615..8729891c 100644 --- a/tests/runconfigs/s1b_los_angeles.yaml +++ b/tests/runconfigs/s1b_los_angeles_mask_off_h5.yaml @@ -53,7 +53,7 @@ runconfig: save_bursts: True save_mosaics: True - output_imagery_format: COG + output_imagery_format: HDF5 output_imagery_compression: ZSTD output_imagery_nbits: 16 @@ -118,7 +118,7 @@ runconfig: input_terrain_radiometry: beta0 # OPTIONAL - Minimum RTC area factor in dB - rtc_min_value_db: + rtc_min_value_db: -30 # RTC DEM upsampling dem_upsampling: 1 @@ -138,14 +138,11 @@ runconfig: # OPTIONAL - Choices: "single_block", "geogrid", "geogrid_radargrid", and "auto" (default) memory_mode: auto - # OPTIONAL - Processing upsampling factor applied to input geogrid - geogrid_upsampling: 1 - # Save the incidence angle save_incidence_angle: False # Save the local-incidence angle - save_local_inc_angle: False + save_local_inc_angle: True # Save the projection angle save_projection_angle: False diff --git a/tests/runconfigs/s1b_los_angeles_mask_on.yaml b/tests/runconfigs/s1b_los_angeles_mask_on.yaml new file mode 100644 index 00000000..bf790a4e --- /dev/null +++ b/tests/runconfigs/s1b_los_angeles_mask_on.yaml @@ -0,0 +1,211 @@ +runconfig: + name: rtc_s1_workflow_default + + groups: + primary_executable: + product_type: RTC_S1 + + pge_name_group: + pge_name: RTC_S1_PGE + + input_file_group: + # Required. List of SAFE files (min=1) + safe_file_path: [data/s1b_los_angeles/input_dir/S1B_IW_SLC__1SDV_20200926T135152_20200926T135219_023547_02CBCC_F988-CROPPED.zip] + + # Required. List of orbit (EOF) files (min=1) + orbit_file_path: [data/s1b_los_angeles/input_dir/S1B_OPER_AUX_PREORB_OPOD_20200926T111139_V20200926T095426_20200926T162926.EOF] + + # Optional. Burst ID to process (empty for all bursts) + # burst_id: [t069_147173_iw1, t069_147174_iw1] + burst_id: [t071_151225_iw1, t071_151226_iw1] + + dynamic_ancillary_file_group: + # Digital elevation model + dem_file: data/s1b_los_angeles/input_dir/dem.tif + + # Digital elevation model description + dem_file_description: + + static_ancillary_file_group: + + # burst database sqlite file + burst_database_file: + + product_group: + processing_type: 'CUSTOM' + + # Directory where PGE will place results + product_path: . + + # Directory where SAS writes temporary data + scratch_path: data/s1b_los_angeles/scratch_dir + + # If option `save_bursts` is set, output bursts are saved to: + # {output_dir}/{burst_id}/{product_id}{suffix}.{ext} + # If option `save_mosaics` is set, output mosaics are saved to: + # {output_dir}/{product_id}{suffix}.{ext} + # If the field `product_id`` is left empty, the prefix "rtc_product" + # will be used instead. + # `suffix` is only used when there are multiple output files. + # `ext` is determined by geocoding_options.output_imagery_format. + output_dir: data/s1b_los_angeles/output_dir + product_id: OPERA_L2_RTC-S1_071-151230-IW2_20200926T135152Z_20230125T134122Z_S1B_30 + + save_bursts: True + save_mosaics: True + output_imagery_format: COG + output_imagery_compression: ZSTD + output_imagery_nbits: 16 + + # Optional. Save secondary layers (e.g., inc. angle) within + # the HDF5 file + save_secondary_layers_as_hdf5: False + + # Save RTC-S1 metadata in the HDF5 format + # Optional for `output_imagery_format` equal to 'ENVI', 'GTiff', or + # 'COG', and enabled by default for `output_imagery_format` equal + # to 'HDF5' or 'NETCDF' or `save_secondary_layers_as_hdf5` is True + save_metadata: True + + processing: + + # Check if ancillary input cover entirely output products + check_ancillary_inputs_coverage: True + + polarization: co-pol + + # Options to run geo2rdr + geo2rdr: + threshold: 1.0e-8 + numiter: 25 + + # Options to run rdr2geo + rdr2geo: + threshold: 1.0e-7 + numiter: 25 + + # Apply absolute radiometric correction + apply_absolute_radiometric_correction: True + + # Apply thermal noise correction + apply_thermal_noise_correction: True + + # OPTIONAL - Apply RTC + apply_rtc: True + + # Apply bistatic delay correction + apply_bistatic_delay_correction: True + + # Apply static tropospheric delay correction + apply_static_tropospheric_delay_correction: True + + # OPTIONAL - to control behavior of RTC module + # (only applicable if geocode.apply_rtc is True) + rtc: + # OPTIONAL - Choices: + # "gamma0" (default) + # "sigma0" + output_type: gamma0 + + # OPTIONAL - Choices: + # "bilinear_distribution" (default) + # "area_projection" + algorithm_type: area_projection + + # OPTIONAL - Choices: + # "beta0" (default) + # "sigma0" + input_terrain_radiometry: beta0 + + # OPTIONAL - Minimum RTC area factor in dB + rtc_min_value_db: -30 + + # RTC DEM upsampling + dem_upsampling: 1 + + # OPTIONAL - Mechanism to specify output posting and DEM + geocoding: + + # OPTIONAL - Apply valid-samples sub-swath masking + apply_valid_samples_sub_swath_masking: True + + # OPTIONAL - Apply shadow masking + apply_shadow_masking: False + + # OPTIONAL - + algorithm_type: area_projection + + # OPTIONAL - Choices: "single_block", "geogrid", "geogrid_radargrid", and "auto" (default) + memory_mode: auto + + # Save the incidence angle + save_incidence_angle: False + + # Save the local-incidence angle + save_local_inc_angle: True + + # Save the projection angle + save_projection_angle: False + + # Save the RTC ANF compuated with the projection angle method + save_rtc_anf_projection_angle: False + + # Save the range slope angle + save_range_slope: False + + # Save the number of looks used to compute RTC-S1 + save_nlooks: True + + # Save the RTC area normalization factor (ANF) used to generate + # the RTC product + save_rtc_anf: True + + # Save the RTC area normalization factor (ANF) gamma0 to sigma0 + save_rtc_anf_gamma0_to_sigma0: False + + # Save interpolated DEM used to compute RTC-S1 + save_dem: False + + # Save layover shadow mask + save_mask: True + + # OPTIONAL - Absolute radiometric correction + abs_rad_cal: 1 + + # OPTIONAL - Clip values above threshold + clip_max: + + # OPTIONAL - Clip values below threshold + clip_min: + + # OPTIONAL - Double sampling of the radar-grid + # input sampling in the range direction + upsample_radargrid: False + + bursts_geogrid: + output_epsg: + x_posting: 300 + y_posting: 300 + x_snap: 300 + y_snap: 300 + top_left: + x: + y: + bottom_right: + x: + y: + + + mosaicking: + mosaic_geogrid: + output_epsg: + x_posting: 300 + y_posting: 300 + x_snap: 300 + y_snap: 300 + top_left: + x: + y: + bottom_right: + x: + y: diff --git a/tests/runconfigs/s1b_los_angeles_mask_on_h5.yaml b/tests/runconfigs/s1b_los_angeles_mask_on_h5.yaml new file mode 100644 index 00000000..115c0491 --- /dev/null +++ b/tests/runconfigs/s1b_los_angeles_mask_on_h5.yaml @@ -0,0 +1,211 @@ +runconfig: + name: rtc_s1_workflow_default + + groups: + primary_executable: + product_type: RTC_S1 + + pge_name_group: + pge_name: RTC_S1_PGE + + input_file_group: + # Required. List of SAFE files (min=1) + safe_file_path: [data/s1b_los_angeles/input_dir/S1B_IW_SLC__1SDV_20200926T135152_20200926T135219_023547_02CBCC_F988-CROPPED.zip] + + # Required. List of orbit (EOF) files (min=1) + orbit_file_path: [data/s1b_los_angeles/input_dir/S1B_OPER_AUX_PREORB_OPOD_20200926T111139_V20200926T095426_20200926T162926.EOF] + + # Optional. Burst ID to process (empty for all bursts) + # burst_id: [t069_147173_iw1, t069_147174_iw1] + burst_id: [t071_151225_iw1, t071_151226_iw1] + + dynamic_ancillary_file_group: + # Digital elevation model + dem_file: data/s1b_los_angeles/input_dir/dem.tif + + # Digital elevation model description + dem_file_description: + + static_ancillary_file_group: + + # burst database sqlite file + burst_database_file: + + product_group: + processing_type: 'CUSTOM' + + # Directory where PGE will place results + product_path: . + + # Directory where SAS writes temporary data + scratch_path: data/s1b_los_angeles/scratch_dir + + # If option `save_bursts` is set, output bursts are saved to: + # {output_dir}/{burst_id}/{product_id}{suffix}.{ext} + # If option `save_mosaics` is set, output mosaics are saved to: + # {output_dir}/{product_id}{suffix}.{ext} + # If the field `product_id`` is left empty, the prefix "rtc_product" + # will be used instead. + # `suffix` is only used when there are multiple output files. + # `ext` is determined by geocoding_options.output_imagery_format. + output_dir: data/s1b_los_angeles/output_dir + product_id: OPERA_L2_RTC-S1_071-151230-IW2_20200926T135152Z_20230125T134122Z_S1B_30 + + save_bursts: True + save_mosaics: True + output_imagery_format: HDF5 + output_imagery_compression: ZSTD + output_imagery_nbits: 16 + + # Optional. Save secondary layers (e.g., inc. angle) within + # the HDF5 file + save_secondary_layers_as_hdf5: True + + # Save RTC-S1 metadata in the HDF5 format + # Optional for `output_imagery_format` equal to 'ENVI', 'GTiff', or + # 'COG', and enabled by default for `output_imagery_format` equal + # to 'HDF5' or 'NETCDF' or `save_secondary_layers_as_hdf5` is True + save_metadata: True + + processing: + + # Check if ancillary input cover entirely output products + check_ancillary_inputs_coverage: True + + polarization: co-pol + + # Options to run geo2rdr + geo2rdr: + threshold: 1.0e-8 + numiter: 25 + + # Options to run rdr2geo + rdr2geo: + threshold: 1.0e-7 + numiter: 25 + + # Apply absolute radiometric correction + apply_absolute_radiometric_correction: True + + # Apply thermal noise correction + apply_thermal_noise_correction: True + + # OPTIONAL - Apply RTC + apply_rtc: True + + # Apply bistatic delay correction + apply_bistatic_delay_correction: True + + # Apply static tropospheric delay correction + apply_static_tropospheric_delay_correction: True + + # OPTIONAL - to control behavior of RTC module + # (only applicable if geocode.apply_rtc is True) + rtc: + # OPTIONAL - Choices: + # "gamma0" (default) + # "sigma0" + output_type: gamma0 + + # OPTIONAL - Choices: + # "bilinear_distribution" (default) + # "area_projection" + algorithm_type: area_projection + + # OPTIONAL - Choices: + # "beta0" (default) + # "sigma0" + input_terrain_radiometry: beta0 + + # OPTIONAL - Minimum RTC area factor in dB + rtc_min_value_db: -30 + + # RTC DEM upsampling + dem_upsampling: 1 + + # OPTIONAL - Mechanism to specify output posting and DEM + geocoding: + + # OPTIONAL - Apply valid-samples sub-swath masking + apply_valid_samples_sub_swath_masking: True + + # OPTIONAL - Apply shadow masking + apply_shadow_masking: False + + # OPTIONAL - + algorithm_type: area_projection + + # OPTIONAL - Choices: "single_block", "geogrid", "geogrid_radargrid", and "auto" (default) + memory_mode: auto + + # Save the incidence angle + save_incidence_angle: False + + # Save the local-incidence angle + save_local_inc_angle: True + + # Save the projection angle + save_projection_angle: False + + # Save the RTC ANF compuated with the projection angle method + save_rtc_anf_projection_angle: False + + # Save the range slope angle + save_range_slope: False + + # Save the number of looks used to compute RTC-S1 + save_nlooks: True + + # Save the RTC area normalization factor (ANF) used to generate + # the RTC product + save_rtc_anf: True + + # Save the RTC area normalization factor (ANF) gamma0 to sigma0 + save_rtc_anf_gamma0_to_sigma0: False + + # Save interpolated DEM used to compute RTC-S1 + save_dem: False + + # Save layover shadow mask + save_mask: True + + # OPTIONAL - Absolute radiometric correction + abs_rad_cal: 1 + + # OPTIONAL - Clip values above threshold + clip_max: + + # OPTIONAL - Clip values below threshold + clip_min: + + # OPTIONAL - Double sampling of the radar-grid + # input sampling in the range direction + upsample_radargrid: False + + bursts_geogrid: + output_epsg: + x_posting: 300 + y_posting: 300 + x_snap: 300 + y_snap: 300 + top_left: + x: + y: + bottom_right: + x: + y: + + + mosaicking: + mosaic_geogrid: + output_epsg: + x_posting: 300 + y_posting: 300 + x_snap: 300 + y_snap: 300 + top_left: + x: + y: + bottom_right: + x: + y: diff --git a/tests/test_rtc_s1_workflow.py b/tests/test_rtc_s1_workflow.py index a0c8aac9..02095c69 100644 --- a/tests/test_rtc_s1_workflow.py +++ b/tests/test_rtc_s1_workflow.py @@ -123,7 +123,7 @@ def _check_results(output_dir, product_prefix, save_imagery_as_hdf5, else: # assert that the following secondary layers are present: - ds_list = ['number_of_looks', 'rtc_area_normalization_factor', + ds_list = ['number_of_looks', 'rtc_anf_gamma0_to_beta0', # 'rtc_area_normalization_factor_gamma0_to_sigma0', 'local_incidence_angle'] for ds_name in ds_list: @@ -154,11 +154,9 @@ def test_workflow(): tests_dir = os.path.dirname(__file__) dataset_dir = os.path.join(test_data_directory, dataset_name) - user_runconfig_file = os.path.join(tests_dir, 'runconfigs', - 's1b_los_angeles.yaml') + if FLAG_ALWAYS_DOWNLOAD or not os.path.isdir(dataset_dir): + - if (FLAG_ALWAYS_DOWNLOAD or not os.path.isdir(dataset_dir) or - not os.path.isfile(user_runconfig_file)): print(f'Test dataset {dataset_name} not found. Downloading' f' file {dataset_url}.') @@ -179,32 +177,20 @@ def test_workflow(): full_log_formatting = False create_logger(log_file, full_log_formatting) - # Get a runconfig dict from command line argumens - runconfig_path = os.path.join(tests_dir, 'runconfigs', - 's1b_los_angeles.yaml') + for runconfig_mode in ['mask_off', 'mask_on', + 'mask_off_h5', 'mask_on_h5']: - # for output_imagery_format in ['COG', 'HDF5']: - for output_imagery_format in ['COG']: + # Get a runconfig dict from command line argumens + runconfig_path = os.path.join( + tests_dir, 'runconfigs', + f's1b_los_angeles_{runconfig_mode}.yaml') cfg = RunConfig.load_from_yaml(runconfig_path) - cfg.groups.product_group.output_imagery_format = output_imagery_format output_dir_single_job, product_prefix, save_imagery_as_hdf5, \ save_secondary_layers_as_hdf5, save_metadata, \ hdf5_file_extension, imagery_extension = _load_cfg_parameters(cfg) - # Testing creation of secondary layers: - # - # The YAML file above (`runconfig_path`) is only set to create the - # `numberOfLooks` and `rtcAreaNormalizationFactorGamma0ToBeta0`. Here, - # we also force the creation of `layoverShadowMask` and - # `localIncidenceAngle` and will test if they are present in the - # output files. We also assert that layers not set to be created, - # such as `incidenceAngle`, are indeed not created - - cfg.groups.processing.geocoding.save_mask = True - cfg.groups.processing.geocoding.save_local_inc_angle = True - # Run geocode burst workflow (single job) run_single_job(cfg)