From fe9dd4fb72f29113c299856fb43604ace1566acb Mon Sep 17 00:00:00 2001 From: BecCowley Date: Fri, 8 Dec 2023 14:55:21 +1100 Subject: [PATCH 1/8] Bug fixes for start & stop depths in histories and depth in global atts. Also some handing updates for older SYST and RCT codes. --- .../DELAYED/xbt_dm_imos_conversion.py | 57 ++++++++++--------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py b/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py index 8bea0923..d85e0913 100755 --- a/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py +++ b/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py @@ -120,22 +120,18 @@ def get_recorder_type(netcdf_file_path): """ return Recorder as defined in WMO4770 """ + rct_list = read_section_from_xbt_config('RCT$') + syst_list = read_section_from_xbt_config('SYST') + with Dataset(netcdf_file_path, 'r', format='NETCDF4') as netcdf_file_obj: gatts = parse_srfc_codes(netcdf_file_path) - # if the file is old and recorder information is from SYST surface code, use SYST list - att_name = 'XBT_system_type' - if att_name in list(gatts.keys()): - rct_list = read_section_from_xbt_config('SYST') - #and change the key name to 'XBT_RECORDER_TYPE' - gatts['XBT_recorder_type'] = gatts['XBT_system_type'] - del gatts['XBT_system_type'] - else: - rct_list = read_section_from_xbt_config('RCT$') - att_name = 'XBT_recorder_type' if att_name in list(gatts.keys()): item_val = str(int(gatts[att_name])) + if item_val in list(syst_list.keys()): + item_val=syst_list[item_val].split(',')[0] + if item_val in list(rct_list.keys()): return item_val, rct_list[item_val].split(',')[0] else: @@ -562,11 +558,11 @@ def generate_xbt_gatts_nc(gatts, data, annex, output_folder): output_netcdf_obj.date_created = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ") if isinstance(data['DEPTH'], np.ma.MaskedArray): - output_netcdf_obj.geospatial_vertical_min = round(np.ma.MaskedArray.min(data['DEPTH']).item(0),2) - output_netcdf_obj.geospatial_vertical_max = round(np.ma.MaskedArray.max(data['DEPTH']).item(0),2) + output_netcdf_obj.geospatial_vertical_min = np.ma.MaskedArray.min(data['DEPTH']).item(0) + output_netcdf_obj.geospatial_vertical_max = np.ma.MaskedArray.max(data['DEPTH']).item(0) else: - output_netcdf_obj.geospatial_vertical_min = round(min(data['DEPTH']),2) - output_netcdf_obj.geospatial_vertical_max = round(max(data['DEPTH']),2) + output_netcdf_obj.geospatial_vertical_min = min(data['DEPTH']) + output_netcdf_obj.geospatial_vertical_max = max(data['DEPTH']) output_netcdf_obj.geospatial_lat_min = data['LATITUDE'] output_netcdf_obj.geospatial_lat_max = data['LATITUDE'] @@ -751,17 +747,20 @@ def generate_xbt_nc(gatts_ed, data_ed, annex_ed, output_folder, *argv): output_netcdf_obj["HISTORY_DATE"][idx] = history_date_obj[idx] output_netcdf_obj["HISTORY_PARAMETER"][idx] = annex_ed['act_parm'][idx] output_netcdf_obj["HISTORY_PREVIOUS_VALUE"][idx] = annex_ed['previous_val'][idx] - output_netcdf_obj["HISTORY_START_DEPTH"][idx] = annex_ed['aux_id'][idx] output_netcdf_obj["HISTORY_QC_FLAG"][idx] = annex_ed['act_code'][idx] #QC,RE, TE, PE and EF flag applies to entire profile res = annex_ed['act_code'][idx] in act_code_full_profile if res: + output_netcdf_obj["HISTORY_START_DEPTH"][idx] = vals[0] output_netcdf_obj["HISTORY_STOP_DEPTH"][idx] = output_netcdf_obj.geospatial_vertical_max continue # Find stop depth depending on which flags are in place - start_idx = np.int_(np.where(vals == annex_ed['aux_id'][idx])) + start_idx = np.int_(np.where(np.logical_and(vals <= annex_ed['aux_id'][idx]+0.1, vals >= annex_ed['aux_id'][idx]-0.1))) + # make the start depth equal to actual depth in depth array + output_netcdf_obj["HISTORY_START_DEPTH"][idx] = vals[start_idx] + #find next deepest flag depth stop_depth = [i for i in annex_ed['aux_id'] if i > annex_ed['aux_id'][idx]] # if the flag is in act_code_single_point list, then stop depth is same as start @@ -774,7 +773,7 @@ def generate_xbt_nc(gatts_ed, data_ed, annex_ed, output_folder, *argv): res = annex_ed['act_code'][idx] in act_code_next_flag if res: if stop_depth: # if not the last flag, next greatest depth - stop_idx = np.int_(np.where(np.round(vals,2) == np.round(stop_depth[0],2))) + stop_idx = np.int_(np.where(np.logical_and(vals <= stop_depth[0]+0.1, vals >= stop_depth[0]-0.1))) stopdepth = vals[stop_idx-1] output_netcdf_obj["HISTORY_STOP_DEPTH"][idx] = stopdepth else: @@ -793,7 +792,7 @@ def generate_xbt_nc(gatts_ed, data_ed, annex_ed, output_folder, *argv): if flag in [1,2,5]: #single point, same stop depth output_netcdf_obj["HISTORY_STOP_DEPTH"][idx] = annex_ed['aux_id'][idx] elif stop_depth: # if not the last flag, next greatest depth - stop_idx = np.int_(np.where(np.round(vals,2) == np.round(stop_depth[0],2))) + stop_idx = np.int_(np.where(np.logical_and(vals <= stop_depth[0]+0.1, vals >= stop_depth[0]-0.1))) stopdepth = vals[stop_idx-1] output_netcdf_obj["HISTORY_STOP_DEPTH"][idx] = stopdepth else: @@ -836,30 +835,32 @@ def generate_xbt_nc(gatts_ed, data_ed, annex_ed, output_folder, *argv): # sort the flags by depth order to help with histories idx_sort = sorted(range(len(annex_raw['aux_id'])), key=lambda k: annex_raw['aux_id'][k]) - vals = data_raw['DEPTH_RAW'].data - qcvals_temp = data_raw['TEMP_RAW_quality_control'].data - qcvals_depth = data_raw['DEPTH_RAW_quality_control'].data + vals = data_raw['DEPTH'].data + qcvals_temp = data_raw['TEMP_quality_control'].data + qcvals_depth = data_raw['DEPTH_quality_control'].data for idx in idx_sort: # slicing over VLEN variable -> need a for loop output_netcdf_obj["HISTORY_INSTITUTION"][idx] = annex_raw['ident_code'][idx] output_netcdf_obj["HISTORY_STEP"][idx] = annex_raw['prc_code'][idx] - names = read_section_from_xbt_config('VARIOUS') - output_netcdf_obj["HISTORY_SOFTWARE"][idx] = names['HISTORY_SOFTWARE'] + output_netcdf_obj["HISTORY_SOFTWARE"][idx] = get_history_val() output_netcdf_obj["HISTORY_SOFTWARE_RELEASE"][idx] = annex_raw['version_soft'][idx] output_netcdf_obj["HISTORY_DATE"][idx] = history_date_obj[idx] output_netcdf_obj["HISTORY_PARAMETER"][idx] = annex_raw['act_parm'][idx] output_netcdf_obj["HISTORY_PREVIOUS_VALUE"][idx] = annex_raw['previous_val'][idx] - output_netcdf_obj["HISTORY_START_DEPTH"][idx] = annex_raw['aux_id'][idx] output_netcdf_obj["HISTORY_QC_FLAG"][idx] = annex_raw['act_code'][idx] #QC,RE, PE, TE and EF flag applies to entire profile res = annex_raw['act_code'][idx] in act_code_full_profile if res: + output_netcdf_obj["HISTORY_START_DEPTH"][idx] = vals[0] output_netcdf_obj["HISTORY_STOP_DEPTH"][idx] = output_netcdf_obj.geospatial_vertical_max continue # Find stop depth depending on which flags are in place - start_idx = np.int_(np.where(vals == annex_raw['aux_id'][idx])) + start_idx = np.int_(np.where(np.logical_and(vals <= annex_ed['aux_id'][idx]+0.1, vals >= annex_ed['aux_id'][idx]-0.1))) + # make the start depth equal to actual depth in depth array + output_netcdf_obj["HISTORY_START_DEPTH"][idx] = vals[start_idx] + #find next deepest flag depth stop_depth = [i for i in annex_raw['aux_id'] if i > annex_raw['aux_id'][idx]] # if the flag is in act_code_single_point list, then stop depth is same as start @@ -872,7 +873,7 @@ def generate_xbt_nc(gatts_ed, data_ed, annex_ed, output_folder, *argv): res = annex_raw['act_code'][idx] in act_code_next_flag if res: if stop_depth: # if not the last flag, next greatest depth - stop_idx = np.int_(np.where(np.round(vals,2) == np.round(stop_depth[0],2))) + stop_idx = np.int_(np.where(np.logical_and(vals <= stop_depth[0]+0.1, vals >= stop_depth[0]-0.1))) stopdepth = vals[stop_idx-1] output_netcdf_obj["HISTORY_STOP_DEPTH"][idx] = stopdepth else: @@ -891,7 +892,7 @@ def generate_xbt_nc(gatts_ed, data_ed, annex_ed, output_folder, *argv): if flag in [1,2,5]: #single point, same stop depth output_netcdf_obj["HISTORY_STOP_DEPTH"][idx] = annex_raw['aux_id'][idx] elif stop_depth: # if not the last flag, next greatest depth - stop_idx = np.int_(np.where(np.round(vals,2) == np.round(stop_depth[0],2))) + stop_idx = np.int_(np.where(np.logical_and(vals <= stop_depth[0]+0.1, vals >= stop_depth[0]-0.1))) stopdepth = vals[stop_idx-1] output_netcdf_obj["HISTORY_STOP_DEPTH"][idx] = stopdepth else: @@ -967,7 +968,7 @@ def clean_temp_val(netcdf_filepath, annex_ed, *argv): for idx, ii_logic in enumerate(idx_raw_cs_flag): if ii_logic: - idx_val_to_modify = depth_raw_flags_val[idx] == output_netcdf_obj["DEPTH_RAW"][:] + idx_val_to_modify = np.logical_and(output_netcdf_obj["DEPTH_RAW"][:] <= depth_ed_flags_val[idx]+0.1,output_netcdf_obj["DEPTH_RAW"][:] >= depth_ed_flags_val[idx]-0.1) if sum(idx_val_to_modify) > 1: _error("Cleaning TEMP_RAW: more than one depth value matching") #TODO improve msg elif sum(idx_val_to_modify) == 0: From 4023ebf402d598d0c0b2f4162c72291484e361fb Mon Sep 17 00:00:00 2001 From: BecCowley Date: Mon, 11 Dec 2023 11:40:58 +1100 Subject: [PATCH 2/8] Add handling for situation where a history start depth cannot be found. --- SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py b/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py index d85e0913..54d18409 100755 --- a/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py +++ b/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py @@ -738,6 +738,10 @@ def generate_xbt_nc(gatts_ed, data_ed, annex_ed, output_folder, *argv): qcvals_temp = data_ed['TEMP_quality_control'].data qcvals_depth = data_ed['DEPTH_quality_control'].data for idx in idx_sort: + # Find stop depth depending on which flags are in place + start_idx = np.int_(np.where(np.logical_and(vals <= annex_ed['aux_id'][idx]+0.1, vals >= annex_ed['aux_id'][idx]-0.1))) + if np.size(start_idx) == 0: + _error('No matching depth for this history record ' + annex_ed['act_code'][idx]) # slicing over VLEN variable -> need a for loop output_netcdf_obj["HISTORY_INSTITUTION"][idx] = annex_ed['ident_code'][idx] output_netcdf_obj["HISTORY_STEP"][idx] = annex_ed['prc_code'][idx] @@ -755,9 +759,7 @@ def generate_xbt_nc(gatts_ed, data_ed, annex_ed, output_folder, *argv): output_netcdf_obj["HISTORY_START_DEPTH"][idx] = vals[0] output_netcdf_obj["HISTORY_STOP_DEPTH"][idx] = output_netcdf_obj.geospatial_vertical_max continue - - # Find stop depth depending on which flags are in place - start_idx = np.int_(np.where(np.logical_and(vals <= annex_ed['aux_id'][idx]+0.1, vals >= annex_ed['aux_id'][idx]-0.1))) + # make the start depth equal to actual depth in depth array output_netcdf_obj["HISTORY_START_DEPTH"][idx] = vals[start_idx] From d2d47f98801b0dcdf1b0a082a9a15209573e6d55 Mon Sep 17 00:00:00 2001 From: BecCowley Date: Tue, 12 Dec 2023 13:00:01 +1100 Subject: [PATCH 3/8] Bug fix to enable different date formats in the prc_date code. Can be yyyymmdd or ddmmyyyy --- SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py b/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py index 54d18409..a4af1fc2 100755 --- a/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py +++ b/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py @@ -12,6 +12,7 @@ import numpy as np import numpy.ma as ma +import pandas as pd from netCDF4 import Dataset, date2num from generate_netcdf_att import generate_netcdf_att, get_imos_parameter_info @@ -283,7 +284,15 @@ def parse_annex_nc(netcdf_file_path): prc_date = [date.replace(' ','0') for date in prc_date] - prc_date = [datetime.strptime(date, '%Y%m%d') for date in prc_date] + # allow for history dates to be YYYYMMDD or DDMMYYYY + df = pd.DataFrame({'date': prc_date}) + date1 = pd.to_datetime(df['date'], errors='coerce', format='%Y%m%d') + date2 = pd.to_datetime(df['date'], errors='coerce', format='%d%m%Y') + prc_date = date1.fillna(date2).to_list() + + # prc_date = [datetime.strptime(date, '%Y%m%d') for date in prc_date] + + aux_id = netcdf_file_obj['Aux_ID'][0:nhist] # depth value of modified act_parm var modified version_soft = [''.join(chr(x) for x in bytearray(xx)).strip() for xx in netcdf_file_obj['Version'][0:nhist].data if bytearray(xx).strip()] From 4345e96538c612d804ea1fd6312a4e3ec2c9e9d2 Mon Sep 17 00:00:00 2001 From: BecCowley Date: Tue, 12 Dec 2023 13:00:44 +1100 Subject: [PATCH 4/8] Keep record of callsign for platforms that aren't in the AODN vocabulary and use 'unknown' as ship name. --- SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py b/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py index a4af1fc2..d422f1c6 100755 --- a/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py +++ b/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py @@ -562,6 +562,8 @@ def generate_xbt_gatts_nc(gatts, data, annex, output_folder): LOGGER.warning('Vessel call sign %s seems to be wrong. Using the closest match to the AODN vocabulary: %s' % ( gatts['Platform_code'], output_netcdf_obj.Callsign)) else: + output_netcdf_obj.ship_name = 'Unknown' + output_netcdf_obj.Callsign = gatts['Platform_code'] LOGGER.warning('Vessel call sign %s is unknown in AODN vocabulary, Please contact info@aodn.org.au' % gatts[ 'Platform_code']) From 0a6c7d0418de8892758e1a49f3907f0050188f04 Mon Sep 17 00:00:00 2001 From: BecCowley Date: Mon, 22 Jan 2024 15:46:28 +1100 Subject: [PATCH 5/8] Add Probe type correction code --- SOOP/SOOP_XBT/DELAYED/xbt_config | 1 + 1 file changed, 1 insertion(+) diff --git a/SOOP/SOOP_XBT/DELAYED/xbt_config b/SOOP/SOOP_XBT/DELAYED/xbt_config index 02791eb8..d80ecbf7 100755 --- a/SOOP/SOOP_XBT/DELAYED/xbt_config +++ b/SOOP/SOOP_XBT/DELAYED/xbt_config @@ -34,6 +34,7 @@ RE = Repeat Drop PE = Profile position error or correction to latitude and/or longitude TE = Date-Time error or correction to date and/or time NT = No Trace +PR = Probe Type corrected #QC codes that only apply to the place they are put [ACT_CODES_SINGLE_POINT] From 43e263664b6276a3a4a7dd6449deea0be959b853 Mon Sep 17 00:00:00 2001 From: BecCowley Date: Thu, 14 Mar 2024 08:48:45 +1100 Subject: [PATCH 6/8] Add IMOS global atts template --- .../DELAYED/generate_nc_file_att_IMOS | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 SOOP/SOOP_XBT/DELAYED/generate_nc_file_att_IMOS diff --git a/SOOP/SOOP_XBT/DELAYED/generate_nc_file_att_IMOS b/SOOP/SOOP_XBT/DELAYED/generate_nc_file_att_IMOS new file mode 100644 index 00000000..0cf82d14 --- /dev/null +++ b/SOOP/SOOP_XBT/DELAYED/generate_nc_file_att_IMOS @@ -0,0 +1,141 @@ +[global_attributes] +Conventions = CF-1.6,IMOS-1.4 +acknowledgement = Any users of IMOS data are required to clearly acknowledge the source of the material derived from IMOS in the format: "Data was sourced from the Integrated Marine Observing System (IMOS) - IMOS is a national collaborative research infrastructure, supported by the Australian Government." If relevant, also credit other organisations involved in collection of this particular datastream (as listed in "credit" in the metadata record). +author = Besnard, Laurent +author_email = laurent.besnard@utas.edu.au +citation = The citation in a list of references is: "IMOS [year-of-data-download], [Title], [data-access-url], accessed [date-of-access]" +data_centre = Australian Ocean Data Network (AODN) +data_centre_email = info@aodn.org.au +disclaimer = Data, products and services from IMOS are provided "as is" without any warranty as to fitness for a particular purpose. +distribution_statement = Data may be re-used, provided that related metadata explaining the data has been reviewed by the user, and the data is appropriately acknowledged. Data, products and services from IMOS are provided "as is" without any warranty as to fitness for a particular purpose. +featureType = profile +file_version = Level 1 - Quality Controlled Data +file_version_quality_control = Data in this file has been through a quality control procedure. Every data point in this file has an associated quality flag +geospatial_vertical_positive = down +geospatial_vertical_units = metres +institution = SOOP XBT +institution_references = http://www.imos.org.au/soop.html +keywords = Oceans>Ocean Temperature>Sea Water Temperature; Oceans>Bathymetry/Seafloor Topography>Water Depth; Bathythermographs>Expendable Bathythermographs (XBT) +license = http://creativecommons.org/licenses/by/4.0/ +naming_authority = IMOS +principal_investigator = Cowley, Rebecca; CSIRO +project = Integrated Marine Observing System (IMOS) +quality_control_set = 1.0 +references = http://www.meds-sdmm.dfo-mpo.gc.ca/meds/Databases/OCEAN/wmocodes_e.htm; http://www.meds-sdmm.dfo-mpo.gc.ca/meds/Databases/OCEAN/GTSPPcodes_e.htm; http://woce.nodc.noaa.gov/woce_v3/wocedata_1/woce-uot/overview.htm; https://www.nodc.noaa.gov/GTSPP/document/codetbls/gtsppcode.html +source = Expendable Bathythermograph (XBT) +standard_name_vocabulary = NetCDF Climate and Forecast (CF) Metadata Convention Standard Name Table 27 + +[TIME] +calendar = gregorian +units = days since 1950-01-01 00:00:00 UTC +ancillary_variables = TIME_quality_control +axis = T + +[TIME_quality_control] +long_name = quality flags for time +standard_name = time status_flag +quality_control_conventions = IMOS standard flags + +[LATITUDE] +reference_datum = geographical coordinates, WGS84 projection +ancillary_variables = LATITUDE_quality_control +axis = Y + +[LATITUDE_quality_control] +standard_name = latitude status_flag +long_name = quality flags for latitude +quality_control_conventions = IMOS standard flags + +[LONGITUDE] +reference_datum = geographical coordinates, WGS84 projection +ancillary_variables = LONGITUDE_quality_control +axis = X + +[LONGITUDE_quality_control] +standard_name = longitude status_flag +long_name = quality flags for longitude +quality_control_conventions = IMOS standard flags + +[DEPTH] +reference_datum = sea surface +standard_name = depth +longname = depth +units = m +valid_min = -5. +valid_max = 12000. +ancillary_variables = DEPTH_quality_control +axis = Z +positive = down + +[DEPTH_quality_control] +long_name = quality flags for depth +standard_name = depth status_flag +quality_control_conventions = IMOS standard flags + +[TEMP] +long_name = sea_water_temperature +standard_name = sea_water_temperature +units = Celsius +valid_min = -2.5 +valid_max = 40. +coordinates = TIME LATITUDE LONGITUDE DEPTH +ancillary_variables = TEMP_quality_control +axis = Z +positive = down + +[TEMP_quality_control] +long_name = quality flag for sea_water_temperature +standard_name = sea_water_temperature status_flag +quality_control_conventions = IMOS standard flags + +[HISTORY_INSTITUTION] +long_name = Institution which performed action +Conventions = GTSPP IDENT_CODE table + +[HISTORY_STEP] +long_name = Step in data processing +Conventions = GTSPP PRC_CODE table + +[HISTORY_SOFTWARE] +long_name = Name of software which performed action +Conventions = Institution dependent + +[HISTORY_SOFTWARE_RELEASE] +long_name = Version/Release of software which performed action +Conventions = Institution dependent + +[HISTORY_DATE] +long_name = Date the history record was created +calendar = gregorian +units = days since 1950-01-01 00:00:00 UTC +axis = T + +[HISTORY_PARAMETER] +long_name = Parameter that action is performed on +Conventions = GTSPP PC_PROF table + +[HISTORY_START_DEPTH] +long_name = Start depth action applied to +units = m +axis = Z +positive = down + +[HISTORY_STOP_DEPTH] +long_name = End depth action applied to +units = m +axis = Z +positive = down + +[HISTORY_PREVIOUS_VALUE] +long_name = Parameter previous value before action +units = 1 +axis = Z +positive = down + +[HISTORY_QC_FLAG] +long_name = QC flag applied +Conventions = GTSPP ACT_CODE table and CSIRO XBT Cookbook + +[HISTORY_QC_FLAG_DESCRIPTION] +long_name = Description of HISTORY_QC_FLAG +Conventions = GTSPP ACT_CODE table and CSIRO XBT Cookbook From 7f9c4fd833bf6c43332e701d2b3699495d17ddb2 Mon Sep 17 00:00:00 2001 From: BecCowley Date: Fri, 26 Apr 2024 12:12:06 +1000 Subject: [PATCH 7/8] Update HISTORY_PREVIOUS_VALUE to hold strings and adjust attributes. --- SOOP/SOOP_XBT/DELAYED/generate_nc_file_att | 3 --- SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py | 7 ++++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/SOOP/SOOP_XBT/DELAYED/generate_nc_file_att b/SOOP/SOOP_XBT/DELAYED/generate_nc_file_att index 0cf82d14..dcfd8072 100644 --- a/SOOP/SOOP_XBT/DELAYED/generate_nc_file_att +++ b/SOOP/SOOP_XBT/DELAYED/generate_nc_file_att @@ -128,9 +128,6 @@ positive = down [HISTORY_PREVIOUS_VALUE] long_name = Parameter previous value before action -units = 1 -axis = Z -positive = down [HISTORY_QC_FLAG] long_name = QC flag applied diff --git a/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py b/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py index d422f1c6..651467e2 100755 --- a/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py +++ b/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py @@ -302,8 +302,9 @@ def parse_annex_nc(netcdf_file_path): #netcdf_file_obj.variables['Previous_Val'][0:nhist]] if x] #TODO: check this bug. Leave in place for now. - previous_val = [float(x) for x in [''.join(chr(x) for x in bytearray(xx).strip()).rstrip('\x00') for xx in - netcdf_file_obj['Previous_Val'][0:nhist]] if x] + previous_val = [x for x in + [''.join(chr(x) for x in bytearray(xx).strip()).rstrip('\x00') + for xx in netcdf_file_obj.variables['Previous_Val'][0:nhist]] if x] ident_code = [''.join(chr(x) for x in bytearray(xx)).strip() for xx in ident_code if bytearray(xx).strip()] data_type = ''.join(chr(x) for x in bytearray(netcdf_file_obj['Data_Type'][:].data)).strip() @@ -693,7 +694,7 @@ def generate_xbt_nc(gatts_ed, data_ed, annex_ed, output_folder, *argv): output_netcdf_obj.createVariable("HISTORY_PARAMETER", "str", 'N_HISTORY') output_netcdf_obj.createVariable("HISTORY_START_DEPTH", "f", 'N_HISTORY') output_netcdf_obj.createVariable("HISTORY_STOP_DEPTH", "f", 'N_HISTORY') - output_netcdf_obj.createVariable("HISTORY_PREVIOUS_VALUE", "f", 'N_HISTORY') + output_netcdf_obj.createVariable("HISTORY_PREVIOUS_VALUE", "str", 'N_HISTORY') output_netcdf_obj.createVariable("HISTORY_QC_FLAG", "str", 'N_HISTORY') output_netcdf_obj.createVariable("HISTORY_QC_FLAG_DESCRIPTION", "str", 'N_HISTORY') From f03921711e8b46fa18361fec2c16d461f99468fa Mon Sep 17 00:00:00 2001 From: BecCowley Date: Mon, 19 Jan 2026 09:54:35 +1100 Subject: [PATCH 8/8] Handle some bugs for older files. --- SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py b/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py index 651467e2..32c63e7b 100755 --- a/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py +++ b/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py @@ -752,6 +752,9 @@ def generate_xbt_nc(gatts_ed, data_ed, annex_ed, output_folder, *argv): for idx in idx_sort: # Find stop depth depending on which flags are in place start_idx = np.int_(np.where(np.logical_and(vals <= annex_ed['aux_id'][idx]+0.1, vals >= annex_ed['aux_id'][idx]-0.1))) + # if start_idx is empty, check if it is a surface flag (depth=0) and change start_idx to 0 + if np.size(start_idx) == 0 and annex_ed['aux_id'][idx]==0: + start_idx = 0 if np.size(start_idx) == 0: _error('No matching depth for this history record ' + annex_ed['act_code'][idx]) # slicing over VLEN variable -> need a for loop @@ -856,7 +859,7 @@ def generate_xbt_nc(gatts_ed, data_ed, annex_ed, output_folder, *argv): # slicing over VLEN variable -> need a for loop output_netcdf_obj["HISTORY_INSTITUTION"][idx] = annex_raw['ident_code'][idx] output_netcdf_obj["HISTORY_STEP"][idx] = annex_raw['prc_code'][idx] - output_netcdf_obj["HISTORY_SOFTWARE"][idx] = get_history_val() + output_netcdf_obj["HISTORY_SOFTWARE"][idx] = 'CSIRO Quality control cookbook for XBT data v1.1' output_netcdf_obj["HISTORY_SOFTWARE_RELEASE"][idx] = annex_raw['version_soft'][idx] output_netcdf_obj["HISTORY_DATE"][idx] = history_date_obj[idx] output_netcdf_obj["HISTORY_PARAMETER"][idx] = annex_raw['act_parm'][idx]