diff --git a/SOOP/SOOP_XBT/DELAYED/generate_nc_file_att b/SOOP/SOOP_XBT/DELAYED/generate_nc_file_att index 0cf82d14..dcfd8072 100644 --- a/SOOP/SOOP_XBT/DELAYED/generate_nc_file_att +++ b/SOOP/SOOP_XBT/DELAYED/generate_nc_file_att @@ -128,9 +128,6 @@ positive = down [HISTORY_PREVIOUS_VALUE] long_name = Parameter previous value before action -units = 1 -axis = Z -positive = down [HISTORY_QC_FLAG] long_name = QC flag applied diff --git a/SOOP/SOOP_XBT/DELAYED/generate_nc_file_att_IMOS b/SOOP/SOOP_XBT/DELAYED/generate_nc_file_att_IMOS new file mode 100644 index 00000000..0cf82d14 --- /dev/null +++ b/SOOP/SOOP_XBT/DELAYED/generate_nc_file_att_IMOS @@ -0,0 +1,141 @@ +[global_attributes] +Conventions = CF-1.6,IMOS-1.4 +acknowledgement = Any users of IMOS data are required to clearly acknowledge the source of the material derived from IMOS in the format: "Data was sourced from the Integrated Marine Observing System (IMOS) - IMOS is a national collaborative research infrastructure, supported by the Australian Government." If relevant, also credit other organisations involved in collection of this particular datastream (as listed in "credit" in the metadata record). +author = Besnard, Laurent +author_email = laurent.besnard@utas.edu.au +citation = The citation in a list of references is: "IMOS [year-of-data-download], [Title], [data-access-url], accessed [date-of-access]" +data_centre = Australian Ocean Data Network (AODN) +data_centre_email = info@aodn.org.au +disclaimer = Data, products and services from IMOS are provided "as is" without any warranty as to fitness for a particular purpose. +distribution_statement = Data may be re-used, provided that related metadata explaining the data has been reviewed by the user, and the data is appropriately acknowledged. Data, products and services from IMOS are provided "as is" without any warranty as to fitness for a particular purpose. +featureType = profile +file_version = Level 1 - Quality Controlled Data +file_version_quality_control = Data in this file has been through a quality control procedure. Every data point in this file has an associated quality flag +geospatial_vertical_positive = down +geospatial_vertical_units = metres +institution = SOOP XBT +institution_references = http://www.imos.org.au/soop.html +keywords = Oceans>Ocean Temperature>Sea Water Temperature; Oceans>Bathymetry/Seafloor Topography>Water Depth; Bathythermographs>Expendable Bathythermographs (XBT) +license = http://creativecommons.org/licenses/by/4.0/ +naming_authority = IMOS +principal_investigator = Cowley, Rebecca; CSIRO +project = Integrated Marine Observing System (IMOS) +quality_control_set = 1.0 +references = http://www.meds-sdmm.dfo-mpo.gc.ca/meds/Databases/OCEAN/wmocodes_e.htm; http://www.meds-sdmm.dfo-mpo.gc.ca/meds/Databases/OCEAN/GTSPPcodes_e.htm; http://woce.nodc.noaa.gov/woce_v3/wocedata_1/woce-uot/overview.htm; https://www.nodc.noaa.gov/GTSPP/document/codetbls/gtsppcode.html +source = Expendable Bathythermograph (XBT) +standard_name_vocabulary = NetCDF Climate and Forecast (CF) Metadata Convention Standard Name Table 27 + +[TIME] +calendar = gregorian +units = days since 1950-01-01 00:00:00 UTC +ancillary_variables = TIME_quality_control +axis = T + +[TIME_quality_control] +long_name = quality flags for time +standard_name = time status_flag +quality_control_conventions = IMOS standard flags + +[LATITUDE] +reference_datum = geographical coordinates, WGS84 projection +ancillary_variables = LATITUDE_quality_control +axis = Y + +[LATITUDE_quality_control] +standard_name = latitude status_flag +long_name = quality flags for latitude +quality_control_conventions = IMOS standard flags + +[LONGITUDE] +reference_datum = geographical coordinates, WGS84 projection +ancillary_variables = LONGITUDE_quality_control +axis = X + +[LONGITUDE_quality_control] +standard_name = longitude status_flag +long_name = quality flags for longitude +quality_control_conventions = IMOS standard flags + +[DEPTH] +reference_datum = sea surface +standard_name = depth +longname = depth +units = m +valid_min = -5. +valid_max = 12000. +ancillary_variables = DEPTH_quality_control +axis = Z +positive = down + +[DEPTH_quality_control] +long_name = quality flags for depth +standard_name = depth status_flag +quality_control_conventions = IMOS standard flags + +[TEMP] +long_name = sea_water_temperature +standard_name = sea_water_temperature +units = Celsius +valid_min = -2.5 +valid_max = 40. +coordinates = TIME LATITUDE LONGITUDE DEPTH +ancillary_variables = TEMP_quality_control +axis = Z +positive = down + +[TEMP_quality_control] +long_name = quality flag for sea_water_temperature +standard_name = sea_water_temperature status_flag +quality_control_conventions = IMOS standard flags + +[HISTORY_INSTITUTION] +long_name = Institution which performed action +Conventions = GTSPP IDENT_CODE table + +[HISTORY_STEP] +long_name = Step in data processing +Conventions = GTSPP PRC_CODE table + +[HISTORY_SOFTWARE] +long_name = Name of software which performed action +Conventions = Institution dependent + +[HISTORY_SOFTWARE_RELEASE] +long_name = Version/Release of software which performed action +Conventions = Institution dependent + +[HISTORY_DATE] +long_name = Date the history record was created +calendar = gregorian +units = days since 1950-01-01 00:00:00 UTC +axis = T + +[HISTORY_PARAMETER] +long_name = Parameter that action is performed on +Conventions = GTSPP PC_PROF table + +[HISTORY_START_DEPTH] +long_name = Start depth action applied to +units = m +axis = Z +positive = down + +[HISTORY_STOP_DEPTH] +long_name = End depth action applied to +units = m +axis = Z +positive = down + +[HISTORY_PREVIOUS_VALUE] +long_name = Parameter previous value before action +units = 1 +axis = Z +positive = down + +[HISTORY_QC_FLAG] +long_name = QC flag applied +Conventions = GTSPP ACT_CODE table and CSIRO XBT Cookbook + +[HISTORY_QC_FLAG_DESCRIPTION] +long_name = Description of HISTORY_QC_FLAG +Conventions = GTSPP ACT_CODE table and CSIRO XBT Cookbook diff --git a/SOOP/SOOP_XBT/DELAYED/xbt_config b/SOOP/SOOP_XBT/DELAYED/xbt_config index 02791eb8..d80ecbf7 100755 --- a/SOOP/SOOP_XBT/DELAYED/xbt_config +++ b/SOOP/SOOP_XBT/DELAYED/xbt_config @@ -34,6 +34,7 @@ RE = Repeat Drop PE = Profile position error or correction to latitude and/or longitude TE = Date-Time error or correction to date and/or time NT = No Trace +PR = Probe Type corrected #QC codes that only apply to the place they are put [ACT_CODES_SINGLE_POINT] diff --git a/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py b/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py index 8bea0923..32c63e7b 100755 --- a/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py +++ b/SOOP/SOOP_XBT/DELAYED/xbt_dm_imos_conversion.py @@ -12,6 +12,7 @@ import numpy as np import numpy.ma as ma +import pandas as pd from netCDF4 import Dataset, date2num from generate_netcdf_att import generate_netcdf_att, get_imos_parameter_info @@ -120,22 +121,18 @@ def get_recorder_type(netcdf_file_path): """ return Recorder as defined in WMO4770 """ + rct_list = read_section_from_xbt_config('RCT$') + syst_list = read_section_from_xbt_config('SYST') + with Dataset(netcdf_file_path, 'r', format='NETCDF4') as netcdf_file_obj: gatts = parse_srfc_codes(netcdf_file_path) - # if the file is old and recorder information is from SYST surface code, use SYST list - att_name = 'XBT_system_type' - if att_name in list(gatts.keys()): - rct_list = read_section_from_xbt_config('SYST') - #and change the key name to 'XBT_RECORDER_TYPE' - gatts['XBT_recorder_type'] = gatts['XBT_system_type'] - del gatts['XBT_system_type'] - else: - rct_list = read_section_from_xbt_config('RCT$') - att_name = 'XBT_recorder_type' if att_name in list(gatts.keys()): item_val = str(int(gatts[att_name])) + if item_val in list(syst_list.keys()): + item_val=syst_list[item_val].split(',')[0] + if item_val in list(rct_list.keys()): return item_val, rct_list[item_val].split(',')[0] else: @@ -287,7 +284,15 @@ def parse_annex_nc(netcdf_file_path): prc_date = [date.replace(' ','0') for date in prc_date] - prc_date = [datetime.strptime(date, '%Y%m%d') for date in prc_date] + # allow for history dates to be YYYYMMDD or DDMMYYYY + df = pd.DataFrame({'date': prc_date}) + date1 = pd.to_datetime(df['date'], errors='coerce', format='%Y%m%d') + date2 = pd.to_datetime(df['date'], errors='coerce', format='%d%m%Y') + prc_date = date1.fillna(date2).to_list() + + # prc_date = [datetime.strptime(date, '%Y%m%d') for date in prc_date] + + aux_id = netcdf_file_obj['Aux_ID'][0:nhist] # depth value of modified act_parm var modified version_soft = [''.join(chr(x) for x in bytearray(xx)).strip() for xx in netcdf_file_obj['Version'][0:nhist].data if bytearray(xx).strip()] @@ -297,8 +302,9 @@ def parse_annex_nc(netcdf_file_path): #netcdf_file_obj.variables['Previous_Val'][0:nhist]] if x] #TODO: check this bug. Leave in place for now. - previous_val = [float(x) for x in [''.join(chr(x) for x in bytearray(xx).strip()).rstrip('\x00') for xx in - netcdf_file_obj['Previous_Val'][0:nhist]] if x] + previous_val = [x for x in + [''.join(chr(x) for x in bytearray(xx).strip()).rstrip('\x00') + for xx in netcdf_file_obj.variables['Previous_Val'][0:nhist]] if x] ident_code = [''.join(chr(x) for x in bytearray(xx)).strip() for xx in ident_code if bytearray(xx).strip()] data_type = ''.join(chr(x) for x in bytearray(netcdf_file_obj['Data_Type'][:].data)).strip() @@ -557,16 +563,18 @@ def generate_xbt_gatts_nc(gatts, data, annex, output_folder): LOGGER.warning('Vessel call sign %s seems to be wrong. Using the closest match to the AODN vocabulary: %s' % ( gatts['Platform_code'], output_netcdf_obj.Callsign)) else: + output_netcdf_obj.ship_name = 'Unknown' + output_netcdf_obj.Callsign = gatts['Platform_code'] LOGGER.warning('Vessel call sign %s is unknown in AODN vocabulary, Please contact info@aodn.org.au' % gatts[ 'Platform_code']) output_netcdf_obj.date_created = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ") if isinstance(data['DEPTH'], np.ma.MaskedArray): - output_netcdf_obj.geospatial_vertical_min = round(np.ma.MaskedArray.min(data['DEPTH']).item(0),2) - output_netcdf_obj.geospatial_vertical_max = round(np.ma.MaskedArray.max(data['DEPTH']).item(0),2) + output_netcdf_obj.geospatial_vertical_min = np.ma.MaskedArray.min(data['DEPTH']).item(0) + output_netcdf_obj.geospatial_vertical_max = np.ma.MaskedArray.max(data['DEPTH']).item(0) else: - output_netcdf_obj.geospatial_vertical_min = round(min(data['DEPTH']),2) - output_netcdf_obj.geospatial_vertical_max = round(max(data['DEPTH']),2) + output_netcdf_obj.geospatial_vertical_min = min(data['DEPTH']) + output_netcdf_obj.geospatial_vertical_max = max(data['DEPTH']) output_netcdf_obj.geospatial_lat_min = data['LATITUDE'] output_netcdf_obj.geospatial_lat_max = data['LATITUDE'] @@ -686,7 +694,7 @@ def generate_xbt_nc(gatts_ed, data_ed, annex_ed, output_folder, *argv): output_netcdf_obj.createVariable("HISTORY_PARAMETER", "str", 'N_HISTORY') output_netcdf_obj.createVariable("HISTORY_START_DEPTH", "f", 'N_HISTORY') output_netcdf_obj.createVariable("HISTORY_STOP_DEPTH", "f", 'N_HISTORY') - output_netcdf_obj.createVariable("HISTORY_PREVIOUS_VALUE", "f", 'N_HISTORY') + output_netcdf_obj.createVariable("HISTORY_PREVIOUS_VALUE", "str", 'N_HISTORY') output_netcdf_obj.createVariable("HISTORY_QC_FLAG", "str", 'N_HISTORY') output_netcdf_obj.createVariable("HISTORY_QC_FLAG_DESCRIPTION", "str", 'N_HISTORY') @@ -742,6 +750,13 @@ def generate_xbt_nc(gatts_ed, data_ed, annex_ed, output_folder, *argv): qcvals_temp = data_ed['TEMP_quality_control'].data qcvals_depth = data_ed['DEPTH_quality_control'].data for idx in idx_sort: + # Find stop depth depending on which flags are in place + start_idx = np.int_(np.where(np.logical_and(vals <= annex_ed['aux_id'][idx]+0.1, vals >= annex_ed['aux_id'][idx]-0.1))) + # if start_idx is empty, check if it is a surface flag (depth=0) and change start_idx to 0 + if np.size(start_idx) == 0 and annex_ed['aux_id'][idx]==0: + start_idx = 0 + if np.size(start_idx) == 0: + _error('No matching depth for this history record ' + annex_ed['act_code'][idx]) # slicing over VLEN variable -> need a for loop output_netcdf_obj["HISTORY_INSTITUTION"][idx] = annex_ed['ident_code'][idx] output_netcdf_obj["HISTORY_STEP"][idx] = annex_ed['prc_code'][idx] @@ -751,17 +766,18 @@ def generate_xbt_nc(gatts_ed, data_ed, annex_ed, output_folder, *argv): output_netcdf_obj["HISTORY_DATE"][idx] = history_date_obj[idx] output_netcdf_obj["HISTORY_PARAMETER"][idx] = annex_ed['act_parm'][idx] output_netcdf_obj["HISTORY_PREVIOUS_VALUE"][idx] = annex_ed['previous_val'][idx] - output_netcdf_obj["HISTORY_START_DEPTH"][idx] = annex_ed['aux_id'][idx] output_netcdf_obj["HISTORY_QC_FLAG"][idx] = annex_ed['act_code'][idx] #QC,RE, TE, PE and EF flag applies to entire profile res = annex_ed['act_code'][idx] in act_code_full_profile if res: + output_netcdf_obj["HISTORY_START_DEPTH"][idx] = vals[0] output_netcdf_obj["HISTORY_STOP_DEPTH"][idx] = output_netcdf_obj.geospatial_vertical_max continue - - # Find stop depth depending on which flags are in place - start_idx = np.int_(np.where(vals == annex_ed['aux_id'][idx])) + + # make the start depth equal to actual depth in depth array + output_netcdf_obj["HISTORY_START_DEPTH"][idx] = vals[start_idx] + #find next deepest flag depth stop_depth = [i for i in annex_ed['aux_id'] if i > annex_ed['aux_id'][idx]] # if the flag is in act_code_single_point list, then stop depth is same as start @@ -774,7 +790,7 @@ def generate_xbt_nc(gatts_ed, data_ed, annex_ed, output_folder, *argv): res = annex_ed['act_code'][idx] in act_code_next_flag if res: if stop_depth: # if not the last flag, next greatest depth - stop_idx = np.int_(np.where(np.round(vals,2) == np.round(stop_depth[0],2))) + stop_idx = np.int_(np.where(np.logical_and(vals <= stop_depth[0]+0.1, vals >= stop_depth[0]-0.1))) stopdepth = vals[stop_idx-1] output_netcdf_obj["HISTORY_STOP_DEPTH"][idx] = stopdepth else: @@ -793,7 +809,7 @@ def generate_xbt_nc(gatts_ed, data_ed, annex_ed, output_folder, *argv): if flag in [1,2,5]: #single point, same stop depth output_netcdf_obj["HISTORY_STOP_DEPTH"][idx] = annex_ed['aux_id'][idx] elif stop_depth: # if not the last flag, next greatest depth - stop_idx = np.int_(np.where(np.round(vals,2) == np.round(stop_depth[0],2))) + stop_idx = np.int_(np.where(np.logical_and(vals <= stop_depth[0]+0.1, vals >= stop_depth[0]-0.1))) stopdepth = vals[stop_idx-1] output_netcdf_obj["HISTORY_STOP_DEPTH"][idx] = stopdepth else: @@ -836,30 +852,32 @@ def generate_xbt_nc(gatts_ed, data_ed, annex_ed, output_folder, *argv): # sort the flags by depth order to help with histories idx_sort = sorted(range(len(annex_raw['aux_id'])), key=lambda k: annex_raw['aux_id'][k]) - vals = data_raw['DEPTH_RAW'].data - qcvals_temp = data_raw['TEMP_RAW_quality_control'].data - qcvals_depth = data_raw['DEPTH_RAW_quality_control'].data + vals = data_raw['DEPTH'].data + qcvals_temp = data_raw['TEMP_quality_control'].data + qcvals_depth = data_raw['DEPTH_quality_control'].data for idx in idx_sort: # slicing over VLEN variable -> need a for loop output_netcdf_obj["HISTORY_INSTITUTION"][idx] = annex_raw['ident_code'][idx] output_netcdf_obj["HISTORY_STEP"][idx] = annex_raw['prc_code'][idx] - names = read_section_from_xbt_config('VARIOUS') - output_netcdf_obj["HISTORY_SOFTWARE"][idx] = names['HISTORY_SOFTWARE'] + output_netcdf_obj["HISTORY_SOFTWARE"][idx] = 'CSIRO Quality control cookbook for XBT data v1.1' output_netcdf_obj["HISTORY_SOFTWARE_RELEASE"][idx] = annex_raw['version_soft'][idx] output_netcdf_obj["HISTORY_DATE"][idx] = history_date_obj[idx] output_netcdf_obj["HISTORY_PARAMETER"][idx] = annex_raw['act_parm'][idx] output_netcdf_obj["HISTORY_PREVIOUS_VALUE"][idx] = annex_raw['previous_val'][idx] - output_netcdf_obj["HISTORY_START_DEPTH"][idx] = annex_raw['aux_id'][idx] output_netcdf_obj["HISTORY_QC_FLAG"][idx] = annex_raw['act_code'][idx] #QC,RE, PE, TE and EF flag applies to entire profile res = annex_raw['act_code'][idx] in act_code_full_profile if res: + output_netcdf_obj["HISTORY_START_DEPTH"][idx] = vals[0] output_netcdf_obj["HISTORY_STOP_DEPTH"][idx] = output_netcdf_obj.geospatial_vertical_max continue # Find stop depth depending on which flags are in place - start_idx = np.int_(np.where(vals == annex_raw['aux_id'][idx])) + start_idx = np.int_(np.where(np.logical_and(vals <= annex_ed['aux_id'][idx]+0.1, vals >= annex_ed['aux_id'][idx]-0.1))) + # make the start depth equal to actual depth in depth array + output_netcdf_obj["HISTORY_START_DEPTH"][idx] = vals[start_idx] + #find next deepest flag depth stop_depth = [i for i in annex_raw['aux_id'] if i > annex_raw['aux_id'][idx]] # if the flag is in act_code_single_point list, then stop depth is same as start @@ -872,7 +890,7 @@ def generate_xbt_nc(gatts_ed, data_ed, annex_ed, output_folder, *argv): res = annex_raw['act_code'][idx] in act_code_next_flag if res: if stop_depth: # if not the last flag, next greatest depth - stop_idx = np.int_(np.where(np.round(vals,2) == np.round(stop_depth[0],2))) + stop_idx = np.int_(np.where(np.logical_and(vals <= stop_depth[0]+0.1, vals >= stop_depth[0]-0.1))) stopdepth = vals[stop_idx-1] output_netcdf_obj["HISTORY_STOP_DEPTH"][idx] = stopdepth else: @@ -891,7 +909,7 @@ def generate_xbt_nc(gatts_ed, data_ed, annex_ed, output_folder, *argv): if flag in [1,2,5]: #single point, same stop depth output_netcdf_obj["HISTORY_STOP_DEPTH"][idx] = annex_raw['aux_id'][idx] elif stop_depth: # if not the last flag, next greatest depth - stop_idx = np.int_(np.where(np.round(vals,2) == np.round(stop_depth[0],2))) + stop_idx = np.int_(np.where(np.logical_and(vals <= stop_depth[0]+0.1, vals >= stop_depth[0]-0.1))) stopdepth = vals[stop_idx-1] output_netcdf_obj["HISTORY_STOP_DEPTH"][idx] = stopdepth else: @@ -967,7 +985,7 @@ def clean_temp_val(netcdf_filepath, annex_ed, *argv): for idx, ii_logic in enumerate(idx_raw_cs_flag): if ii_logic: - idx_val_to_modify = depth_raw_flags_val[idx] == output_netcdf_obj["DEPTH_RAW"][:] + idx_val_to_modify = np.logical_and(output_netcdf_obj["DEPTH_RAW"][:] <= depth_ed_flags_val[idx]+0.1,output_netcdf_obj["DEPTH_RAW"][:] >= depth_ed_flags_val[idx]-0.1) if sum(idx_val_to_modify) > 1: _error("Cleaning TEMP_RAW: more than one depth value matching") #TODO improve msg elif sum(idx_val_to_modify) == 0: