diff --git a/src/neonutilities/__pycache__/__init__.cpython-311.pyc b/src/neonutilities/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..b304059 Binary files /dev/null and b/src/neonutilities/__pycache__/__init__.cpython-311.pyc differ diff --git a/src/neonutilities/__pycache__/aop_download.cpython-311.pyc b/src/neonutilities/__pycache__/aop_download.cpython-311.pyc new file mode 100644 index 0000000..45a7e0d Binary files /dev/null and b/src/neonutilities/__pycache__/aop_download.cpython-311.pyc differ diff --git a/src/neonutilities/__pycache__/citation.cpython-311.pyc b/src/neonutilities/__pycache__/citation.cpython-311.pyc new file mode 100644 index 0000000..b5d53d2 Binary files /dev/null and b/src/neonutilities/__pycache__/citation.cpython-311.pyc differ diff --git a/src/neonutilities/__pycache__/files_by_uri.cpython-311.pyc b/src/neonutilities/__pycache__/files_by_uri.cpython-311.pyc new file mode 100644 index 0000000..a61e63f Binary files /dev/null and b/src/neonutilities/__pycache__/files_by_uri.cpython-311.pyc differ diff --git a/src/neonutilities/__pycache__/get_issue_log.cpython-311.pyc b/src/neonutilities/__pycache__/get_issue_log.cpython-311.pyc new file mode 100644 index 0000000..c4eb540 Binary files /dev/null and b/src/neonutilities/__pycache__/get_issue_log.cpython-311.pyc differ diff --git a/src/neonutilities/__pycache__/read_table_neon.cpython-311.pyc b/src/neonutilities/__pycache__/read_table_neon.cpython-311.pyc new file mode 100644 index 0000000..fdd4465 Binary files /dev/null and b/src/neonutilities/__pycache__/read_table_neon.cpython-311.pyc differ diff --git a/src/neonutilities/__pycache__/tabular_download.cpython-311.pyc b/src/neonutilities/__pycache__/tabular_download.cpython-311.pyc new file mode 100644 index 0000000..424b40a Binary files /dev/null and b/src/neonutilities/__pycache__/tabular_download.cpython-311.pyc differ diff --git a/src/neonutilities/__pycache__/unzip_and_stack.cpython-311.pyc b/src/neonutilities/__pycache__/unzip_and_stack.cpython-311.pyc new file mode 100644 index 0000000..a3ed8da Binary files /dev/null and b/src/neonutilities/__pycache__/unzip_and_stack.cpython-311.pyc differ diff --git a/src/neonutilities/aop_download.py b/src/neonutilities/aop_download.py index 051885c..215c64d 100644 --- a/src/neonutilities/aop_download.py +++ b/src/neonutilities/aop_download.py @@ -111,7 +111,7 @@ def get_file_urls(urls, token=None): response = get_api(api_url=url, token=token) if response is None: logging.info( - "Data file retrieval failed. Check NEON data portal for outage alerts." + "NEON data file retrieval failed. Check NEON data portal for outage alerts." ) # get release info @@ -180,11 +180,11 @@ def get_shared_flights(site): flightSite = shared_flights_dict[site] if site in ["TREE", "CHEQ", "KONA", "DCFS"]: logging.info( - f"{site} is part of the flight box for {flightSite}. Downloading data from {flightSite}." + f"{site} is part of the NEON flight box for {flightSite}. Downloading data from {flightSite}." ) else: logging.info( - f"{site} is an aquatic site and is sometimes included in the flight box for {flightSite}. Aquatic sites are not always included in the flight coverage every year.\nDownloading data from {flightSite}. Check data to confirm coverage of {site}." + f"{site} is a NEON aquatic site and is sometimes included in the flight box for {flightSite}. Aquatic sites are not always included in the flight coverage every year.\nDownloading data from {flightSite}. Check data to confirm coverage of {site}." ) site = flightSite return site @@ -220,7 +220,7 @@ def validate_dpid(dpid): dpid_pattern = "DP[1-4]{1}.[0-9]{5}.00[1-2]{1}" if not re.fullmatch(dpid_pattern, dpid): raise ValueError( - f"{dpid} is not a properly formatted data product ID. The correct format is DP#.#####.00#" + f"{dpid} is not a properly formatted NEON data product ID. The correct format is DP#.#####.00#" ) @@ -314,13 +314,13 @@ def validate_aop_dpid(dpid): # Check if the dpid matches the pattern if not re.fullmatch(aop_dpid_pattern, dpid): raise ValueError( - f"{dpid} is not a valid AOP data product ID. AOP data products follow the format DP#.300##.00#." + f"{dpid} is not a valid NEON AOP data product ID. AOP data products follow the format DP#.300##.00#." ) # Check if the dpid is in the list of suspended AOP dpids if dpid in suspended_aop_dpids: raise ValueError( - f"{dpid} has been suspended and is not currently available, see https://data.neonscience.org/data-products/{dpid} for more details." + f"NEON {dpid} has been suspended and is not currently available, see https://data.neonscience.org/data-products/{dpid} for more details." ) # ' Valid AOP IDs are: {", ".join(valid_aop_dpids)}.') # Check if the dpid is in the list of valid AOP dpids @@ -328,7 +328,7 @@ def validate_aop_dpid(dpid): valid_aop_dpids.sort() valid_aop_dpids_string = "\n".join(valid_aop_dpids) raise ValueError( - f"{dpid} is not a valid AOP data product ID. Valid AOP IDs are listed below:\n{valid_aop_dpids_string}" + f"NEON {dpid} is not a valid AOP data product ID. Valid AOP IDs are listed below:\n{valid_aop_dpids_string}" ) @@ -345,7 +345,7 @@ def validate_aop_l3_dpid(dpid): # Check if the dpid starts with DP3 if not dpid.startswith("DP3"): raise ValueError( - f"{dpid} is not a valid Level 3 (L3) AOP data product ID. Level 3 AOP products follow the format DP3.300##.00#" + f"NEON {dpid} is not a valid Level 3 (L3) AOP data product ID. Level 3 AOP products follow the format DP3.300##.00#" ) # Check if the dpid is in the list of valid AOP dpids @@ -358,7 +358,7 @@ def validate_aop_l3_dpid(dpid): # f'{key}: {value}' for key, value in dpid_dict.items()) raise ValueError( - f"{dpid} is not a valid Level 3 (L3) AOP data product ID. Valid L3 AOP IDs are listed below:\n{valid_aop_l3_dpids_string}" + f"NEON {dpid} is not a valid Level 3 (L3) AOP data product ID. Valid L3 AOP IDs are listed below:\n{valid_aop_l3_dpids_string}" ) # below prints out the corresponding data product names for each ID. # f'{dpid} is not a valid Level 3 (L3) AOP data product ID. Valid L3 AOP products are listed below.\n{formatted_dpid_dict}') @@ -367,7 +367,7 @@ def validate_aop_l3_dpid(dpid): def check_field_spectra_dpid(dpid): if dpid == "DP1.30012.001": raise ValueError( - f"{dpid} is the Field spectral data product, which is published as tabular data. Use zipsByProduct() or loadByProduct() to download these data." + f"NEON {dpid} is the Field spectral data product, which is published as tabular data. Use zipsByProduct() or loadByProduct() to download these data." ) @@ -375,7 +375,7 @@ def validate_site_format(site): site_pattern = "[A-Z]{4}" if not re.fullmatch(site_pattern, site): raise ValueError( - f"{site} is an invalid site format. A four-letter NEON site code is required. NEON site codes can be found here: https://www.neonscience.org/field-sites/explore-field-sites" + f"{site} is an invalid NEON site format. A four-letter NEON site code is required. NEON site codes can be found here: https://www.neonscience.org/field-sites/explore-field-sites" ) @@ -393,13 +393,13 @@ def validate_year(year): year_pattern = "20[1-9][0-9]" if not re.fullmatch(year_pattern, year): raise ValueError( - f'{year} is an invalid year. Year is required in the format "2017" or 2017, eg. AOP data are available from 2013 to present.' + f'{year} is an invalid year. Year is required in the format "2017" or 2017, eg. NEON AOP data are available from 2013 to present.' ) def check_aop_dpid(response_dict, dpid): if response_dict["data"]["productScienceTeamAbbr"] != "AOP": - logging.info(f"{dpid} is not a remote sensing product. Use zipsByProduct()") + logging.info(f"NEON {dpid} is not a remote sensing product. Use zipsByProduct()") return @@ -468,7 +468,7 @@ def list_available_dates(dpid, site): # if the available_releases variable doesn't exist, this error will show up: # UnboundLocalError: local variable 'available_releases' referenced before assignment raise ValueError( - f"There are no data available for the data product {dpid} at the site {site}." + f"There are no NEON data available for the data product {dpid} at the site {site}." ) @@ -630,7 +630,7 @@ def get_aop_tile_extents(dpid, site, year, token=None): # error message if nothing is available if len(site_year_urls) == 0: logging.info( - f"There are no {dpid} data available at the site {site} in {year}. \nTo display available dates for a given data product and site, use the function list_available_dates()." + f"There are no NEON {dpid} data available at the site {site} in {year}. \nTo display available dates for a given data product and site, use the function list_available_dates()." ) return @@ -771,7 +771,7 @@ def by_file_aop( # error message if nothing is available if len(site_year_urls) == 0: logging.info( - f"There are no {dpid} data available at the site {site} in {year}.\nTo display available dates for a given data product and site, use the function list_available_dates()." + f"There are no NEON {dpid} data available at the site {site} in {year}.\nTo display available dates for a given data product and site, use the function list_available_dates()." ) # print("There are no data available at the selected site and year.") return @@ -782,14 +782,14 @@ def by_file_aop( # get the number of files in the dataframe, if there are no files to download, return if len(file_url_df) == 0: # print("No data files found.") - logging.info("No data files found.") + logging.info("No NEON data files found.") return # if 'PROVISIONAL' in releases and not include_provisional: if include_provisional: # log provisional included message logging.info( - "Provisional data are included. To exclude provisional data, use input parameter include_provisional=False." + "Provisional NEON data are included. To exclude provisional data, use input parameter include_provisional=False." ) else: # log provisional not included message and filter to the released data @@ -798,13 +798,13 @@ def by_file_aop( file_url_df = file_url_df[file_url_df["release"] != "PROVISIONAL"] if len(file_url_df) == 0: logging.info( - "Provisional data are not included. To download provisional data, use input parameter include_provisional=True." + "NEON Provisional data are not included. To download provisional data, use input parameter include_provisional=True." ) num_files = len(file_url_df) if num_files == 0: logging.info( - "No data files found. Available data may all be provisional. To download provisional data, use input parameter include_provisional=True." + "No NEON data files found. Available data may all be provisional. To download provisional data, use input parameter include_provisional=True." ) return @@ -818,7 +818,7 @@ def by_file_aop( if check_size: if ( input( - f"Continuing will download {num_files} files totaling approximately {download_size}. Do you want to proceed? (y/n) " + f"Continuing will download {num_files} NEON data files totaling approximately {download_size}. Do you want to proceed? (y/n) " ) != "y" ): @@ -834,7 +834,7 @@ def by_file_aop( # serially download all files, with progress bar files = list(file_url_df["url"]) - print(f"Downloading {num_files} files totaling approximately {download_size}\n") + print(f"Downloading {num_files} NEON data files totaling approximately {download_size}\n") sleep(1) for file in tqdm(files): download_file( @@ -1036,7 +1036,7 @@ def by_tile_aop( response_dict = response.json() # error message if dpid is not an AOP data product if response_dict["data"]["productScienceTeamAbbr"] != "AOP": - print(f"{dpid} is not a remote sensing product. Use zipsByProduct()") + print(f"NEON {dpid} is not a remote sensing product. Use zipsByProduct()") return # replace collocated site with the site name it's published under @@ -1048,7 +1048,7 @@ def by_tile_aop( # error message if nothing is available if len(site_year_urls) == 0: logging.info( - f"There are no {dpid} data available at the site {site} in {year}.\nTo display available dates for a given data product and site, use the function list_available_dates()." + f"There are no NEON {dpid} data available at the site {site} in {year}.\nTo display available dates for a given data product and site, use the function list_available_dates()." ) return @@ -1057,27 +1057,27 @@ def by_tile_aop( # get the number of files in the dataframe, if there are no files to download, return if len(file_url_df) == 0: - logging.info("No data files found.") + logging.info("No NEON data files found.") return # if 'PROVISIONAL' in releases and not include_provisional: if include_provisional: # print provisional included message logging.info( - "Provisional data are included. To exclude provisional data, use input parameter include_provisional=False." + "Provisional NEON data are included. To exclude provisional data, use input parameter include_provisional=False." ) else: # print provisional not included message file_url_df = file_url_df[file_url_df["release"] != "PROVISIONAL"] logging.info( - "Provisional data are not included. To download provisional data, use input parameter include_provisional=True." + "Provisional NEON data are not included. To download provisional data, use input parameter include_provisional=True." ) # get the number of files in the dataframe after filtering for provisional data, if there are no files to download, return num_files = len(file_url_df) if num_files == 0: logging.info( - "No data files found. Available data may all be provisional. To download provisional data, use input parameter include_provisional=True." + "No NEON data files found. Available data may all be provisional. To download provisional data, use input parameter include_provisional=True." ) return @@ -1092,7 +1092,7 @@ def by_tile_aop( # importlib.import_module('pyproj') except ImportError: logging.info( - "Package pyproj is required for this function to work at the BLAN site. Install and re-try" + "Package pyproj is required for this function to work at the NEON BLAN site. Install and re-try" ) return @@ -1203,7 +1203,7 @@ def get_buffer_coords(easting, northing, buffer): coords_not_found = list(set(coord_strs).difference(list(unique_coords_to_download))) if len(coords_not_found) > 0: print( - "Warning, the following coordinates fall outside the bounds of the site, so will not be downloaded:" + "Warning, the following coordinates fall outside the bounds of the NEON site, so will not be downloaded:" ) for coord in coords_not_found: print(",".join(coord.split("_"))) @@ -1211,7 +1211,7 @@ def get_buffer_coords(easting, northing, buffer): # get the number of files in the dataframe, if there are no files to download, return num_files = len(file_url_df_subset) if num_files == 0: - print("No data files found.") + print("No NEON data files found.") return # get the total size of all the files found @@ -1223,7 +1223,7 @@ def get_buffer_coords(easting, northing, buffer): if check_size: if ( input( - f"Continuing will download {num_files} files totaling approximately {download_size}. Do you want to proceed? (y/n) " + f"Continuing will download {num_files} NEON data files totaling approximately {download_size}. Do you want to proceed? (y/n) " ) != "y" ): @@ -1240,7 +1240,7 @@ def get_buffer_coords(easting, northing, buffer): # serially download all files, with progress bar files = list(file_url_df_subset["url"]) - print(f"Downloading {num_files} files totaling approximately {download_size}\n") + print(f"Downloading {num_files} NEON data files totaling approximately {download_size}\n") sleep(1) for file in tqdm(files): download_file( diff --git a/src/neonutilities/files_by_uri.py b/src/neonutilities/files_by_uri.py new file mode 100644 index 0000000..9e64c8a --- /dev/null +++ b/src/neonutilities/files_by_uri.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import requests +import os +import platform +import logging +import importlib.metadata +import pandas as pd +from tqdm import tqdm +from urllib.parse import urlparse +from .unzip_and_stack import unzip_zipfile +from .helper_mods.metadata_helpers import convert_byte_size + +# Set global user agent +vers = importlib.metadata.version("neonutilities") +plat = platform.python_version() +osplat = platform.platform() +usera = f"neonutilities/{vers} Python/{plat} {osplat}" + +def files_by_uri( + filepath, + savepath=None, + check_size=True, + unzip=True, + save_zipped_files=False, + progress=True, +): + """ + + Get files from NEON GCS Bucket using URLs in stacked data + + Parameters + -------- + filepath: The location of the NEON data containing URIs. Can be either a local directory containing NEON tabular data or a list object containing tabular data. + savepath: The location to save the output files from the GCS bucket, optional. Defaults to creating a "GCS_zipFiles" folder in the filepath directory. + check_size: should the user be told the total file size before downloading, defaults to True? (true/false) + unzip: indicates if the downloaded files from GCS buckets should be unzipped into the same directory, defaults to True. Supports .zip and .tar.gz files currently. (true/false) + save_zipped_files: Should the unzipped monthly data folders be retained, defaults to False? Supports .zip and .tar.gz files currently. (true/false) + progress: Should a progress bar be displayed? + + Return + -------- + A folder in the working directory (or in savepath, if specified), containing all files meeting query criteria. + + Example + -------- + ZN NOTE: Insert example when function is coded + + >>> example + + Created on Fri Aug 9 2024 + + @author: Zachary Nickerson + """ + + # check that filepath points to either a directory or a Python list object + if not isinstance(filepath, dict): + if not os.path.exists(filepath): + raise Exception( + "Input filepath is not a list object in the environment nor an existing file directory." + ) + + # if filepath is a dictionary, make a savepath + if isinstance(filepath, dict): + tabList = filepath + if savepath is not None: + if not os.path.exists(savepath): + try: + os.makedirs(os.path.join(savepath, "GCS_files")) + except OSError: + print( + f"Could not create savepath directory. NEON files will be saved to {os.getcwd()}/GCS_files" + ) + savepath = f"{os.getcwd()}" + else: + # if filepath is a directory, read in contents + files = [ + os.path.join(filepath, f) + for f in os.listdir(filepath) + if os.path.isfile(os.path.join(filepath, f)) + ] + files = [file for file in files if file.endswith(".csv")] + tabList = {} + for j, file in enumerate(files): + try: + tabList[file] = pd.read_csv(file) + except Exception: + print(f"File {file} could not be read.") + tabList[f"error{j}"] = None + continue + tabList = {k: v for k, v in tabList.items() if not k.startswith("error")} + + # Check for the variables file in the filepath + varList = [k for k in tabList.keys() if "variables" in k] + if len(varList) == 0: + raise Exception("NEON Variables file was not found in specified filepath.") + if len(varList) > 1: + raise Exception("More than one NEON variables file found in filepath.") + varFile = tabList[varList[0]] + + URLs = varFile[varFile["dataType"] == "uri"] + + # All of the tables in the package with URLs to download + allTables = URLs["table"].unique() + + # Loop through tables and fields to compile a list of URLs to download + URLsToDownload = [] + + # Remove allTables values that aren't in tabList + allTables = {key for key in tabList.keys() for substr in allTables if substr in key} + + if len(allTables) < 1: + raise Exception("No NEON tables with URIs available in download package contents.") + + # Loop through tables + for table in allTables: + tableData = tabList[table] + # Find URLs per table that are in URLs.fieldName + URLsPerTable = [url for url in tableData if url in URLs["fieldName"].values] + # Append the URLs from the fields found + URLsToDownload.extend([tableData[url] for url in URLsPerTable]) + + # Remove duplicates from the list of URLs + uniqueURLs = set() + for lst in URLsToDownload: + uniqueURLs.update(lst) + URLsToDownload = uniqueURLs + + # Remove None values + URLsToDownload = [url for url in URLsToDownload if url is not None] + URLsToDownload = [url for url in URLsToDownload if str(url) != "nan"] + URLsToDownload = [url for url in URLsToDownload if str(url) != ""] + + if len(URLsToDownload) == 0: + raise Exception("There are no NEON URLs other than 'None' for the stacked data.") + + # Create savepath only if it does not already exist + if savepath is not None: + savepath = os.path.join(savepath, "GCS_files") + else: + print( + f"Could not create savepath directory. NEON files will be saved to {os.getcwd()}/GCS_files" + ) + savepath = f"{os.getcwd()}/GCS_files" + if not os.path.exists(os.path.join(savepath)): + os.makedirs(savepath) + + # Check the existence and size of each file from URL + if check_size: + logging.info(f"Checking size of downloading {len(URLsToDownload)} files by URI") + sz = [] + for urlfile in tqdm(URLsToDownload, disable=not progress): + response = requests.head( + urlfile, headers={"User-Agent": usera}, allow_redirects=True + ) + + # Return nothing if response failed + if response.status_code != 200: + logging.info( + "Connection error for a subset of urls. Check outputs for missing data." + ) + # return None + + # Compile file sizes + flszi = int(response.headers["Content-Length"]) + sz.append(flszi) + + # Check download size + download_size = convert_byte_size(sum(sz)) + if ( + input( + f"Continuing will download {len(URLsToDownload)} NEON files totaling approximately {download_size}. Do you want to proceed? (y/n) " + ) + != "y" + ): + logging.info("Download halted.") + return None + else: + logging.info( + f"Downloading {len(URLsToDownload)} NEON files totaling approximately {download_size}." + ) + + # Download URLs + for j in tqdm(URLsToDownload, disable=not progress): + parsed_url = urlparse(j) + filename = os.path.basename(parsed_url.path) + file_path = os.path.join(savepath, filename) + # Download the file + response = requests.get(j, headers={"User-Agent": usera}) + with open(file_path, "wb") as out_file: + out_file.write(response.content) + # If the file type is zip and unzip is True, unzip + if unzip is True: + if file_path.endswith(".zip"): + unzip_zipfile(zippath=file_path) + # Remove zip files after being unzipped + if save_zipped_files is False: + os.remove(file_path) diff --git a/src/neonutilities/helper_mods/__pycache__/__init__.cpython-311.pyc b/src/neonutilities/helper_mods/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..7d1c64e Binary files /dev/null and b/src/neonutilities/helper_mods/__pycache__/__init__.cpython-311.pyc differ diff --git a/src/neonutilities/helper_mods/__pycache__/api_helpers.cpython-311.pyc b/src/neonutilities/helper_mods/__pycache__/api_helpers.cpython-311.pyc new file mode 100644 index 0000000..c3b09cc Binary files /dev/null and b/src/neonutilities/helper_mods/__pycache__/api_helpers.cpython-311.pyc differ diff --git a/src/neonutilities/helper_mods/__pycache__/metadata_helpers.cpython-311.pyc b/src/neonutilities/helper_mods/__pycache__/metadata_helpers.cpython-311.pyc new file mode 100644 index 0000000..0f35667 Binary files /dev/null and b/src/neonutilities/helper_mods/__pycache__/metadata_helpers.cpython-311.pyc differ diff --git a/src/neonutilities/tabular_download.py b/src/neonutilities/tabular_download.py index 86d45f0..5afac02 100644 --- a/src/neonutilities/tabular_download.py +++ b/src/neonutilities/tabular_download.py @@ -62,7 +62,7 @@ def query_files( if package == "expanded": if not lst["data"]["productHasExpanded"]: logging.info( - "No expanded package found for " + "No expanded package found for NEON " + dpid + ". Basic package downloaded instead." ) @@ -121,7 +121,7 @@ def query_files( ) qreq = get_api(api_url=qurl, token=token) if qreq is None: - logging.info("No API response for selected query. Check inputs.") + logging.info("No NEON Portal API response for selected query. Check inputs.") return None qdict = qreq.json() @@ -267,38 +267,38 @@ def zips_by_product( # error message if dpid is not formatted correctly if not re.search(pattern="DP[1-4]{1}.[0-9]{5}.00[0-9]{1}", string=dpid): raise ValueError( - f"{dpid} is not a properly formatted data product ID. The correct format is DP#.#####.00#" + f"{dpid} is not a properly formatted NEON data product ID. The correct format is DP#.#####.00#" ) # error message if package is not basic or expanded if not package in ["basic", "expanded"]: raise ValueError( - f"{package} is not a valid package name. Package must be basic or expanded" + f"{package} is not a valid NEON download package name. Package must be basic or expanded" ) # error messages for products that can't be downloaded by zips_by_product() # AOP products if dpid[4:5:1] == 3 and dpid != "DP1.30012.001": raise ValueError( - f"{dpid} is a remote sensing data product. Use the by_file_aop() or by_tile_aop() function." + f"{dpid} is a NEON remote sensing data product. Use the by_file_aop() or by_tile_aop() function." ) # Phenocam products if dpid == "DP1.00033.001" or dpid == "DP1.00042.001": raise ValueError( - f"{dpid} is a phenological image product, data are hosted by Phenocam." + f"{dpid} is a NEON phenological image product, data are hosted by Phenocam." ) # Aeronet product if dpid == "DP1.00043.001": raise ValueError( - f"Spectral sun photometer ({dpid}) data are hosted by Aeronet." + f"NEON Spectral sun photometer ({dpid}) data are hosted by Aeronet." ) # DHP expanded package if dpid == "DP1.10017.001" and package == "expanded": raise ValueError( - "Digital hemispherical images expanded file packages exceed programmatic download limits. Either download from the data portal, or download the basic package and use the URLs in the data to download the images themselves. Follow instructions in the Data Product User Guide for image file naming." + "NEON Digital hemispherical images expanded file packages exceed programmatic download limits. Either download from the data portal, or download the basic package and use the URLs in the data to download the images themselves. Follow instructions in the Data Product User Guide for image file naming." ) # individual SAE products @@ -325,17 +325,17 @@ def zips_by_product( "DP1.00030.001", ]: raise ValueError( - f"{dpid} is only available in the bundled eddy covariance data product. Download DP4.00200.001 to access these data." + f"{dpid} is only available in the bundled NEON eddy covariance data product. Download DP4.00200.001 to access these data." ) # check for incompatible values of release= and include_provisional= if release == "PROVISIONAL" and not include_provisional: raise ValueError( - "Download request is for release=PROVISIONAL. To download PROVISIONAL data, enter input parameter include_provisional=True." + "NEON Download request is for release=PROVISIONAL. To download PROVISIONAL data, enter input parameter include_provisional=True." ) if re.search(pattern="RELEASE", string=release) is not None and include_provisional: logging.info( - f"Download request is for release={release} but include_provisional=True. Only data in {release} will be downloaded." + f"NEON Download request is for release={release} but include_provisional=True. Only data in {release} will be downloaded." ) # error message if dates aren't formatted correctly @@ -379,7 +379,7 @@ def zips_by_product( siter.append(sx) if indx == 1: logging.info( - f"Some sites in your download request are aquatic sites where {dpid} is collected at a nearby terrestrial site. The sites you requested, and the sites that will be accessed instead, are listed below." + f"Some NEON sites in your download request are aquatic sites where {dpid} is collected at a nearby terrestrial site. The sites you requested, and the sites that will be accessed instead, are listed below." ) logging.info(f"{s} -> {''.join(sx)}") else: @@ -399,7 +399,7 @@ def zips_by_product( ) if newDPID == ["depends"]: raise ValueError( - "Root chemistry and isotopes have been bundled with the root biomass data. For root chemistry from Megapits, download DP1.10066.001. For root chemistry from periodic sampling, download DP1.10067.001." + "NEON Root chemistry and isotopes have been bundled with the root biomass data. For root chemistry from Megapits, download DP1.10066.001. For root chemistry from periodic sampling, download DP1.10067.001." ) else: raise ValueError( @@ -418,7 +418,7 @@ def zips_by_product( other_bundles_df["homeProduct"][other_bundles_df["product"] == dpid] ) raise ValueError( - f"In all releases after {bundle_release}, {''.join(dpid)} has been bundled with {''.join(newDPID)} and is not available independently. Please download {''.join(newDPID)}." + f"In all NEON releases after {bundle_release}, {''.join(dpid)} has been bundled with {''.join(newDPID)} and is not available independently. Please download {''.join(newDPID)}." ) # end of error and exception handling, start the work @@ -439,7 +439,7 @@ def zips_by_product( if prodreq is None: if release == "LATEST": logging.info( - f"No data found for product {dpid}. LATEST data requested; check that token is valid for LATEST access." + f"No data found for NEON data product {dpid}. LATEST data requested; check that token is valid for LATEST access." ) return else: @@ -449,7 +449,7 @@ def zips_by_product( ) if rels is None: raise ConnectionError( - "Data product was not found or API was unreachable." + f"NEON Data product {dpid} was not found or API was unreachable." ) relj = rels.json() reld = relj["data"] @@ -457,14 +457,14 @@ def zips_by_product( for i in range(0, len(reld)): rellist.append(reld[i]["release"]) if release not in rellist: - raise ValueError(f"Release not found. Valid releases are {rellist}") + raise ValueError(f"Release not found. Valid NEON data releases are {rellist}") else: raise ConnectionError( - "Data product was not found or API was unreachable." + f"NEON Data product {dpid} was not found or API was unreachable." ) else: raise ConnectionError( - "Data product was not found or API was unreachable." + f"NEON Data product {dpid} was not found or API was unreachable." ) avail = prodreq.json() @@ -473,7 +473,7 @@ def zips_by_product( # I think this would never be called due to the way get_api() is set up try: avail["error"]["status"] - logging.info(f"No data found for product {dpid}") + logging.info(f"No NEON data found for data product {dpid}") return except Exception: pass @@ -481,7 +481,7 @@ def zips_by_product( # check that token was used if "x-ratelimit-limit" in prodreq.headers and token is not None: if prodreq.headers.get("x-ratelimit-limit") == 200: - logging.info("API token was not recognized. Public rate limit applied.") + logging.info("NEON Portal API token was not recognized. Public rate limit applied.") # use query endpoint if cloud mode selected if cloud_mode: @@ -508,7 +508,7 @@ def zips_by_product( # check for no results if len(month_urls) == 0: - logging.info("There are no data matching the search criteria.") + logging.info(f"There are no NEON {dpid} data matching the search criteria.") return # un-nest list @@ -527,7 +527,7 @@ def zips_by_product( # check for no results if len(site_urls) == 0: - logging.info("There are no data at the selected sites.") + logging.info(f"There are no NEON {dpid} data at the selected sites.") return # subset by start date @@ -541,7 +541,7 @@ def zips_by_product( # check for no results if len(start_urls) == 0: - logging.info("There are no data at the selected date(s).") + logging.info(f"There are no NEON {dpid} data at the selected date(s).") return # subset by end date @@ -553,7 +553,7 @@ def zips_by_product( # check for no results if len(end_urls) == 0: - logging.info("There are no data at the selected date(s).") + logging.info(f"There are no NEON {dpid} data at the selected date(s).") return # if downloading entire site-months, pass to get_zip_urls to query each month for url @@ -584,7 +584,7 @@ def zips_by_product( if check_size: if ( input( - f"Continuing will download {len(durls['z'])} files totaling approximately {download_size}. Do you want to proceed? (y/n) " + f"Continuing will download {len(durls['z'])} NEON {dpid} files totaling approximately {download_size}. Do you want to proceed? (y/n) " ) != "y" ): @@ -592,7 +592,7 @@ def zips_by_product( return None else: logging.info( - f"Downloading {len(durls['z'])} files totaling approximately {download_size}." + f"Downloading {len(durls['z'])} NEON {dpid} files totaling approximately {download_size}." ) # set up folder to save to diff --git a/src/neonutilities/unzip_and_stack.py b/src/neonutilities/unzip_and_stack.py index bc92220..7913d71 100644 --- a/src/neonutilities/unzip_and_stack.py +++ b/src/neonutilities/unzip_and_stack.py @@ -801,7 +801,7 @@ def stack_data_files_parallel(folder, package, dpid, progress=True, cloud_mode=F # stack frame files if progress: logging.info( - "Stacking per-sample files. These files may be very large; download data in smaller subsets if performance problems are encountered.\n" + f"Stacking NEON {dpid} per-sample files. These files may be very large; download data in smaller subsets if performance problems are encountered.\n" ) # subset microbe community data by taxonomic group @@ -842,7 +842,7 @@ def stack_data_files_parallel(folder, package, dpid, progress=True, cloud_mode=F # if there are no datafiles, exit if len(datafls) == 0: - print("No data files are present in specified file path.\n") + print(f"No NEON {dpid} data files are present in specified file path.\n") return None # if there is one or more than one file, stack files @@ -993,7 +993,7 @@ def stack_data_files_parallel(folder, package, dpid, progress=True, cloud_mode=F # set to string if variables file can't be found tableschema = None logging.info( - f"Variables file not found for table {j}. Data types will be inferred if possible." + f"NEON {dpid} variables file not found for table {j}. Data types will be inferred if possible." ) else: tableschema = get_variables(tablepkgvar) @@ -1205,7 +1205,7 @@ def stack_data_files_parallel(folder, package, dpid, progress=True, cloud_mode=F ) if len(rs) > 1: logging.info( - "Multiple data releases were stacked together. This is not appropriate, check your input data." + f"Multiple NEON data releases were stacked together for {dpid}. This is not appropriate, check your input data." ) except Exception: pass @@ -1290,7 +1290,7 @@ def stack_by_table( # Error handling if there are no standardized NEON Portal data tables in the list of files if not any(re.search(r"NEON.D[0-9]{2}.[A-Z]{4}.", x) for x in files): - logging.info("Data files are not present in the specified filepath.") + logging.info("NEON data files are not present in the specified filepath.") return # Determine dpid @@ -1304,7 +1304,7 @@ def stack_by_table( dpid = list(set(dpid)) if not len(dpid) == 1: logging.info( - "Data product ID could not be determined. Check that filepath contains data files, from a single NEON data product." + "NEON Data product ID could not be determined. Check that filepath contains data files, from a single NEON data product." ) return else: @@ -1324,14 +1324,14 @@ def stack_by_table( # Error message for AOP data if dpid[4] == "3" and not dpid == "DP1.30012.001": logging.info( - "This is an AOP data product, files cannot be stacked. Use by_file_aop() or by_tile_aop() to download data." + "This is a NEON AOP data product, files cannot be stacked. Use by_file_aop() or by_tile_aop() to download data." ) return # Error messafe for SAE data if dpid == "DP4.00200.001": logging.info( - "This eddy covariance data product is in HDF5 format. Stack using the stackEddy() function in the R package version of neonUtilities." + "This NEON eddy covariance data product is in HDF5 format. Stack using the stackEddy() function in the R package version of neonUtilities." ) return @@ -1339,14 +1339,14 @@ def stack_by_table( if dpid == "DP1.10017.001" and package == "expanded": save_unzipped_files = True logging.info( - "Note: Digital hemispheric photos (in NEF format) cannot be stacked; only the CSV metadata files will be stacked." + "Note: NEON Digital hemispheric photos (in NEF format) cannot be stacked; only the CSV metadata files will be stacked." ) # Warning about all sensor soil data # Test and modify the file length for the alert, this should be a lot better with arrow if dpid in ["DP1.00094.001", "DP1.00041.001"] and len(files) > 24: logging.info( - "Warning! Attempting to stack soil sensor data. Note that due to the number of soil sensors at each site, data volume is very high for these data. Consider dividing data processing into chunks and/or using a high-performance system." + "Warning! Attempting to stack NEON soil sensor data. Note that due to the number of soil sensors at each site, data volume is very high for these data. Consider dividing data processing into chunks and/or using a high-performance system." ) # If all checks pass, unzip and stack files diff --git a/testdata/NEON_uri_testdata/00131_allSites_2022_RELEASE2025/geo_surveySummary.csv b/testdata/NEON_uri_testdata/00131_allSites_2022_RELEASE2025/geo_surveySummary.csv new file mode 100644 index 0000000..4d6c53b --- /dev/null +++ b/testdata/NEON_uri_testdata/00131_allSites_2022_RELEASE2025/geo_surveySummary.csv @@ -0,0 +1,29 @@ +"uid","domainID","siteID","namedLocation","startDate","surveyEndDate","samplingImpractical","eventID","surveyBoutTypeID","meanBankfullWidth","S1habitatID","S2habitatID","sensorSetLength","sensorSetSlope","sensorSetElevDiff","thalwegLength","thalwegSlope","channelSinuosityIndex","percentDry","percentComplete","percentSubterranean","totalTransformationError","samplingProtocolVersion","sopVersion","dataFileName","dataFilePath","rawDataFileName","rawDataFilePath","remarks","processedSurveyVersion","dataQF","publicationDate","release" +"9b36a79f-99be-4e0f-81c8-75ca899c4a58","D01","HOPB","HOPB",2022-08-08 20:00:00,2022-08-08 20:00:00,"OK","HOPB.202208","rapid habitat",NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,"NEON.DOC.003162vD",NA,NA,NA,"NEON_D01_HOPB_20220808_RAPID_HABITAT_L0_VG.zip","https://storage.neonscience.org/neon-geobath-files/NEON_D01_HOPB_20220808_RAPID_HABITAT_L0_VG.zip",NA,"VG",NA,"20241213T143301Z","RELEASE-2025" +"228b83f9-0e1b-4443-bc49-e736d44c0354","D02","LEWI","LEWI",2022-06-07 20:00:00,2022-06-07 20:00:00,"OK","LEWI.202206","rapid habitat",NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,"NEON.DOC.003162vD",NA,NA,NA,"NEON_D02_LEWI_20220607_RAPID_HABITAT_L0_VG.zip","https://storage.neonscience.org/neon-geobath-files/NEON_D02_LEWI_20220607_RAPID_HABITAT_L0_VG.zip",NA,"VG",NA,"20241213T142715Z","RELEASE-2025" +"1a848c11-ed35-409e-b066-e921ba30a660","D02","POSE","POSE",2022-06-07 20:00:00,2022-06-07 20:00:00,"OK","POSE.202206","rapid habitat",NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,"NEON.DOC.003162vD",NA,NA,NA,"NEON_D02_POSE_20220620_RAPID_HABITAT_L0_VG.zip","https://storage.neonscience.org/neon-geobath-files/NEON_D02_POSE_20220620_RAPID_HABITAT_L0_VG.zip",NA,"VG",NA,"20241213T142923Z","RELEASE-2025" +"5a656852-6371-433b-9951-82214f06fcbf","D04","CUPE","CUPE",2022-04-06 12:47:00,2022-04-06 12:47:00,"OK","CUPE.202204","rapid habitat",NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,"NEON.DOC.003162vD",NA,NA,NA,"NEON_D04_CUPE_20220406_RAPID_HABITAT_L0_VG.zip","https://storage.neonscience.org/neon-geobath-files/NEON_D04_CUPE_20220406_RAPID_HABITAT_L0_VG.zip",NA,"VG",NA,"20241218T012308Z","RELEASE-2025" +"05cffe16-b6b9-4ce8-bce1-eaaecc197019","D04","CUPE","CUPE",2022-09-29 14:49:00,2022-09-29 14:49:00,"OK","CUPE.202209","AIS survey",NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,"NEON.DOC.005399vA",NA,NA,NA,NA,NA,NA,NA,NA,"20241213T142946Z","RELEASE-2025" +"44d1bca6-0d2c-4e1d-9f6b-a0631387f8f6","D04","GUIL","GUIL",2022-04-18 14:53:00,2022-05-26 17:14:00,"OK","GUIL.202205","geomorphology",10.57,"riffle11","pool5",233.35,1.55,3.61,1130.01,1.05,0.79,0,100,0,0,"NEON.DOC.003162vA",NA,"NEON_D04_GUIL_GEOMORPH_20220526_L4_VG.zip","https://storage.neonscience.org/neon-geobath-files/NEON_D04_GUIL_GEOMORPH_20220526_L4_VG.zip",NA,NA,NA,"VG",NA,"20241213T143345Z","RELEASE-2025" +"ecfc8eca-1b30-45b5-8ac5-962eccc7ec64","D06","KING","KING",2022-09-14 16:34:00,2022-09-14 23:22:00,"OK","KING.202209","AIS survey",NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,"NEON.DOC.005399vA",NA,NA,NA,NA,NA,NA,NA,NA,"20241213T143012Z","RELEASE-2025" +"32e9beca-881b-45a1-b04e-910eec9f7002","D06","KING","KING",2022-11-30 17:34:00,2022-12-01 17:41:00,"OK","KING.202212","rapid habitat",NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,"NEON.DOC.003162vD",NA,NA,NA,"NEON_D06_KING_20221201_RAPID_HABITAT_PROCESSED_L0_VG.zip","https://storage.neonscience.org/neon-geobath-files/NEON_D06_KING_20221201_RAPID_HABITAT_PROCESSED_L0_VG.zip",NA,"VG",NA,"20250120T170317Z","RELEASE-2025" +"2e348d79-bd76-476f-b375-97ce20eae4ba","D06","MCDI","MCDI",2022-12-01 18:00:00,2022-12-01 18:00:00,"other","MCDI.202212","rapid habitat",NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,"NEON.DOC.003162vD",NA,NA,NA,NA,NA,"missing survey data",NA,NA,"20241221T162825Z","RELEASE-2025" +"853b0198-9e84-4aab-8919-9819c932a8fa","D07","LECO","LECO",2022-10-05 22:00:00,2022-10-05 22:00:00,"OK","LECO.202210","rapid habitat",NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,"NEON.DOC.003162vD",NA,NA,NA,"NEON_D07_LECO_20221005_RAPID_HABITAT_L0_VG.zip","https://storage.neonscience.org/neon-geobath-files/NEON_D07_LECO_20221005_RAPID_HABITAT_L0_VG.zip",NA,"VG",NA,"20241213T143144Z","RELEASE-2025" +"ba6a1f4c-2256-43cb-a153-a0edc9077080","D07","WALK","WALK",2022-08-22 14:39:00,2022-09-15 12:27:00,"OK","WALK.202209","geomorphology",4.17,"pool5","run18",159.95,1.99,3.19,1239.29,2.9,0.79,0,100,0,0,"NEON.DOC.003162vA",NA,"NEON_D07_WALK_GEOMORPH_20220915_L4_VG.zip","https://storage.neonscience.org/neon-geobath-files/NEON_D07_WALK_GEOMORPH_20220915_L4_VG.zip",NA,NA,NA,"VG",NA,"20241213T142611Z","RELEASE-2025" +"9f0fdc51-5d28-4421-bf05-105a45eef4be","D08","MAYF","MAYF",2022-01-17 15:50:00,2022-01-17 21:29:00,"OK","MAYF.202201","AIS survey",NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,"NEON.DOC.005399vA",NA,NA,NA,NA,NA,NA,NA,NA,"20241213T143507Z","RELEASE-2025" +"d498e8fe-c1fd-4c9f-b355-34b2e987bdae","D08","MAYF","MAYF",2022-11-22 15:36:00,2022-12-07 15:24:00,"OK","MAYF.202212","geomorphology",7.81,"pool6","pool5",232.68,0.55,1.27,1537.42,0.37,0.77,0,100,0,0,"NEON.DOC.003162vA",NA,"NEON_D08_MAYF_GEOMORPH_20221207_L4_VG.zip","https://storage.neonscience.org/neon-geobath-files/NEON_D08_MAYF_GEOMORPH_20221207_L4_VG.zip",NA,NA,NA,"VG",NA,"20241213T143444Z","RELEASE-2025" +"ece45c6f-3e49-4362-86ac-03a98006dbae","D10","ARIK","ARIK",2022-03-01 15:31:00,2022-03-03 18:32:00,"OK","ARIK.202203","geomorphology",5.18,"pool4","pool3",167.52,0.46,0.78,1198.21,0.32,1.35,0,100,0,0,"NEON.DOC.003162vA",NA,"NEON_D10_ARIK_GEOMORPH_20220303_L4_VG.zip","https://storage.neonscience.org/neon-geobath-files/NEON_D10_ARIK_GEOMORPH_20220303_L4_VG.zip",NA,NA,NA,"VG",NA,"20241213T142715Z","RELEASE-2025" +"2ae24649-b1f5-4b51-b27a-b62bad939571","D11","BLUE","BLUE",2022-07-25 22:00:00,2022-07-25 22:00:00,"OK","BLUE.202207","rapid habitat",NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,"NEON.DOC.003162vD",NA,NA,NA,"NEON_D11_BLUE_20220725_RAPID_HABITAT_L0_VG.zip","https://storage.neonscience.org/neon-geobath-files/NEON_D11_BLUE_20220725_RAPID_HABITAT_L0_VG.zip",NA,"VG",NA,"20241213T143459Z","RELEASE-2025" +"8ad533d9-df8c-4a1c-933d-35f1d53236d6","D13","COMO","COMO",2022-08-08 16:02:00,2022-08-31 18:04:00,"OK","COMO.202208","geomorphology",2.88,"riffle4","step-pool3",91.19,7.77,7.09,1168.04,8.73,0.69,0,100,0,0,"NEON.DOC.003162vA",NA,"NEON_D13_COMO_GEOMORPH_20220831_L4_VG.zip","https://storage.neonscience.org/neon-geobath-files/NEON_D13_COMO_GEOMORPH_20220831_L4_VG.zip",NA,NA,NA,"VG",NA,"20250110T035201Z","RELEASE-2025" +"73a2c7a7-15bf-4743-a827-c5085e3aaec2","D14","SYCA","SYCA",2022-03-01 16:00:00,2022-03-15 21:23:00,"OK","SYCA.202203","AIS survey",NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,"NEON.DOC.005399vA",NA,NA,NA,NA,NA,NA,NA,NA,"20241213T142824Z","RELEASE-2025" +"641cd7e5-16c2-4f50-a220-5bcf7ff7bb20","D14","SYCA","SYCA",2022-05-25 02:00:00,2022-05-25 02:00:00,"OK","SYCA.202205","rapid habitat",NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,"NEON.DOC.003162vD",NA,NA,NA,"NEON_D14_SYCA_20220524_RAPID_HABITAT_L0_VG.zip","https://storage.neonscience.org/neon-geobath-files/NEON_D14_SYCA_20220524_RAPID_HABITAT_L0_VG.zip",NA,"VG",NA,"20241213T142618Z","RELEASE-2025" +"bb2c20fb-4506-4e46-a5e0-8004e0e59600","D15","REDB","REDB",2022-11-08 17:40:00,2022-11-30 17:49:00,"OK","REDB.202211","geomorphology",3.83,"pool1","step-pool3",300.02,5.64,16.93,769.36,4.65,0.73,0,70.35,0,0,"NEON.DOC.003162vD",NA,"NEON_D15_REDB_GEOMORPH_20221130_L4_VG.zip","https://storage.neonscience.org/neon-geobath-files/NEON_D15_REDB_GEOMORPH_20221130_L4_VG.zip",NA,NA,NA,"VG",NA,"20250113T223951Z","RELEASE-2025" +"82ca5a1e-1e5e-4c53-8524-f290b8f167b2","D16","MART","MART",2022-07-13 02:00:00,2022-07-13 02:00:00,"OK","MART.202207","rapid habitat",NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,"NEON.DOC.003162vD",NA,NA,NA,"NEON_D16_MART_20220712_RAPID_HABITAT_L0_VG.zip","https://storage.neonscience.org/neon-geobath-files/NEON_D16_MART_20220712_RAPID_HABITAT_L0_VG.zip",NA,"VG",NA,"20241213T143753Z","RELEASE-2025" +"3316e44b-0c60-4e43-b79e-f7c2c2e453f6","D16","MCRA","MCRA",2022-08-17 02:00:00,2022-08-17 02:00:00,"OK","MCRA.202208","rapid habitat",NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,"NEON.DOC.003162vD",NA,NA,NA,"NEON_D16_MCRA_20220816_RAPID_HABITAT_L0_VG.zip","https://storage.neonscience.org/neon-geobath-files/NEON_D16_MCRA_20220816_RAPID_HABITAT_L0_VG.zip",NA,"VG",NA,"20241213T142550Z","RELEASE-2025" +"fd5dda51-2221-4ec9-b635-c944b730d513","D16","MCRA","MCRA",2022-08-30 17:18:00,2022-08-31 21:16:00,"OK","MCRA.202208","AIS survey",NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,"NEON.DOC.005399vA",NA,NA,NA,NA,NA,NA,NA,NA,"20241213T142550Z","RELEASE-2025" +"33031176-c591-459a-8884-3ea82494b514","D17","BIGC","BIGC",2022-01-25 18:21:00,2022-01-26 21:41:00,"OK","BIGC.202201","AIS survey",NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,"NEON.DOC.005399vA",NA,NA,NA,NA,NA,NA,NA,NA,"20241213T142344Z","RELEASE-2025" +"7e5cc402-a3df-43e9-9eb4-7369eccf71c0","D17","BIGC","BIGC",2022-08-24 16:40:00,2022-08-24 16:40:00,"OK","BIGC.202208","rapid habitat",NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,"NEON.DOC.003162vD",NA,NA,NA,"NEON_D17_BIGC_20220824_RAPID_HABITAT_L0_VG.zip","https://storage.neonscience.org/neon-geobath-files/NEON_D17_BIGC_20220824_RAPID_HABITAT_L0_VG.zip",NA,"VG",NA,"20241218T011357Z","RELEASE-2025" +"4ee9bb34-76ce-4616-b91b-40315fd8f5cc","D17","TECR","TECR",2022-08-31 02:00:00,2022-08-31 02:00:00,"OK","TECR.202208","rapid habitat",NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,"NEON.DOC.003162vD",NA,NA,NA,"NEON_D17_TECR_20220830_RAPID_HABITAT_L0_VG.zip","https://storage.neonscience.org/neon-geobath-files/NEON_D17_TECR_20220830_RAPID_HABITAT_L0_VG.zip",NA,"VG",NA,"20241213T142642Z","RELEASE-2025" +"cee4749e-3a79-4f16-8255-78230bff4401","D18","OKSR","OKSR",2022-07-12 04:00:00,2022-07-12 04:00:00,"OK","OKSR.202207","rapid habitat",NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,"NEON.DOC.003162vD",NA,NA,NA,"NEON_D18_OKSR_20220711_RAPID_HABITAT_L0_VG.zip","https://storage.neonscience.org/neon-geobath-files/NEON_D18_OKSR_20220711_RAPID_HABITAT_L0_VG.zip",NA,"VG",NA,"20241213T142644Z","RELEASE-2025" +"2f9cf2ad-748b-420c-b962-193448735d91","D18","OKSR","OKSR",2022-08-10 20:52:00,2022-08-10 20:52:00,"OK","OKSR.202208","AIS survey",NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,"NEON.DOC.005399vA",NA,NA,NA,NA,NA,NA,NA,NA,"20241213T142621Z","RELEASE-2025" +"f7de0faa-856c-46df-a169-173e6c769e2b","D19","CARI","CARI",2022-07-11 20:53:00,2022-09-01 19:44:00,"OK","CARI.202209","geomorphology",2.9,"run23","run13",382.45,1.09,4.16,1353.26,1.07,0.99,0,100,0.22,0.32,"NEON.DOC.003162vA",NA,"NEON_D19_CARI_GEOMORPH_20220901_L4_VG.zip","https://storage.neonscience.org/neon-geobath-files/NEON_D19_CARI_GEOMORPH_20220901_L4_VG.zip","NEON_D19_CARI_GEOMORPH_20220901_L0_VG.zip","https://storage.neonscience.org/neon-geobath-files/NEON_D19_CARI_GEOMORPH_20220901_L0_VG.zip",NA,"VG",NA,"20241213T142352Z","RELEASE-2025" diff --git a/testdata/NEON_uri_testdata/00131_allSites_2022_RELEASE2025/variables_00131.csv b/testdata/NEON_uri_testdata/00131_allSites_2022_RELEASE2025/variables_00131.csv new file mode 100644 index 0000000..18f5a50 --- /dev/null +++ b/testdata/NEON_uri_testdata/00131_allSites_2022_RELEASE2025/variables_00131.csv @@ -0,0 +1,342 @@ +"table","fieldName","description","dataType","units","downloadPkg","pubFormat","primaryKey","categoricalCodeName" +"geo_featureInfo","uid","Unique ID within NEON database; an identifier for the record","string",NA,"basic","asIs","N","" +"geo_featureInfo","domainID","Unique identifier of the NEON domain","string",NA,"basic","asIs","N","" +"geo_featureInfo","siteID","NEON site code","string",NA,"basic","asIs","N","" +"geo_featureInfo","namedLocation","Name of the measurement location in the NEON database","string",NA,"basic","asIs","N","" +"geo_featureInfo","totalStationLocation","Identifier for sequential total station locations throughout the survey","string",NA,"basic","asIs","N","" +"geo_featureInfo","startDate","The start date-time or interval during which an event occurred","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_featureInfo","surveyEndDate","The date-time indicating the end of all surveying activities","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_featureInfo","eventID","An identifier for the set of information associated with the event, which includes information about the place and time of the event","string",NA,"basic","asIs","Y","" +"geo_featureInfo","featureID","Identifier for the transect named location","string",NA,"basic","asIs","Y","" +"geo_featureInfo","featureType","The type of feature surveyed at a given total station location","string",NA,"basic","asIs","Y","" +"geo_featureInfo","habitatType","Habitat type sampled","string",NA,"basic","LOV","N","geo.usdsHabitatType" +"geo_featureInfo","bankfullWidth","The measured bankfull width of the cross-section transect","real","meter","basic","*.#(round)","N","" +"geo_featureInfo","bankfullDepth","The measured bankfull depth of the transect","real","meter","basic","*.#(round)","N","" +"geo_featureInfo","floodproneHeight","The measured floodprone height of the transect; two times the bankfull depth","real","meter","basic","*.#(round)","N","" +"geo_featureInfo","staffGaugeInstalled","Indicator of whether a staff gauge is installed at the site","string",NA,"basic","LOV","N","Yes or No choice" +"geo_featureInfo","staffGaugeMark","The graduated mark on the staff gauge used for mapping","real","meter","basic","*.#(round)","N","" +"geo_featureInfo","pebbleCountsCollected","Indicator of whether a pebble count survey was conducted at the transect","string",NA,"basic","LOV","N","Yes or No choice" +"geo_featureInfo","dataQF","Data quality flag","string",NA,"basic","asIs","N","" +"geo_geomorphicFeatureCount","uid","Unique ID within NEON database; an identifier for the record","string",NA,"basic","asIs","N","" +"geo_geomorphicFeatureCount","domainID","Unique identifier of the NEON domain","string",NA,"basic","asIs","N","" +"geo_geomorphicFeatureCount","siteID","NEON site code","string",NA,"basic","asIs","N","" +"geo_geomorphicFeatureCount","namedLocation","Name of the measurement location in the NEON database","string",NA,"basic","asIs","N","" +"geo_geomorphicFeatureCount","startDate","The start date-time or interval during which an event occurred","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_geomorphicFeatureCount","surveyEndDate","The date-time indicating the end of all surveying activities","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_geomorphicFeatureCount","eventID","An identifier for the set of information associated with the event, which includes information about the place and time of the event","string",NA,"basic","asIs","Y","" +"geo_geomorphicFeatureCount","geomorphicFeatureType","The geomorphic feature type mapped during the survey","string",NA,"basic","LOV","Y","geo.geomorphicFeatureType" +"geo_geomorphicFeatureCount","geomorphicFeatureCount","Total number of unique features identified for a given featureType","integer","number","basic","integer","N","" +"geo_geomorphicFeatureCount","processedSurveyVersion","The version of the post-processing method that can be linked to the specific survey data package stored in the cloud","string",NA,"basic","LOV","N","geo.processedSurveyVersion" +"geo_geomorphicFeatureCount","remarks","Technician notes; free text comments accompanying the record","string",NA,"basic","asIs","N","" +"geo_geomorphicFeatureCount","dataQF","Data quality flag","string",NA,"basic","asIs","N","" +"geo_mappedPointErrors","uid","Unique ID within NEON database; an identifier for the record","string",NA,"expanded","asIs","N","" +"geo_mappedPointErrors","domainID","Unique identifier of the NEON domain","string",NA,"expanded","asIs","N","" +"geo_mappedPointErrors","siteID","NEON site code","string",NA,"expanded","asIs","N","" +"geo_mappedPointErrors","namedLocation","Name of the measurement location in the NEON database","string",NA,"expanded","asIs","N","" +"geo_mappedPointErrors","startDate","The start date-time or interval during which an event occurred","dateTime",NA,"expanded","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_mappedPointErrors","surveyEndDate","The date-time indicating the end of all surveying activities","dateTime",NA,"expanded","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_mappedPointErrors","eventID","An identifier for the set of information associated with the event, which includes information about the place and time of the event","string",NA,"expanded","asIs","Y","" +"geo_mappedPointErrors","pointID","Identifier for a point location","string",NA,"expanded","asIs","Y","" +"geo_mappedPointErrors","deletePoint","Indicator of whether a point is to be deleted during post-processing","string",NA,"expanded","LOV","N","Yes or No choice" +"geo_mappedPointErrors","errorDescription","Description of the type of error associated with the mapped point","string",NA,"expanded","LOV","Y","geo.errorDescription" +"geo_mappedPointErrors","errorDescriptionRemarks","Technician notes; free text comments that describe the error record","string",NA,"expanded","asIs","N","" +"geo_mappedPointErrors","dataQF","Data quality flag","string",NA,"expanded","asIs","N","" +"geo_misclosureInfo","uid","Unique ID within NEON database; an identifier for the record","string",NA,"expanded","asIs","N","" +"geo_misclosureInfo","domainID","Unique identifier of the NEON domain","string",NA,"expanded","asIs","N","" +"geo_misclosureInfo","siteID","NEON site code","string",NA,"expanded","asIs","N","" +"geo_misclosureInfo","namedLocation","Name of the measurement location in the NEON database","string",NA,"expanded","asIs","N","" +"geo_misclosureInfo","startDate","The start date-time or interval during which an event occurred","dateTime",NA,"expanded","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_misclosureInfo","surveyEndDate","The date-time indicating the end of all surveying activities","dateTime",NA,"expanded","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_misclosureInfo","eventID","An identifier for the set of information associated with the event, which includes information about the place and time of the event","string",NA,"expanded","asIs","Y","" +"geo_misclosureInfo","loopClosureNumber","Number of the survey loop closure","integer","number","expanded","integer","Y","" +"geo_misclosureInfo","benchmarkID","Identifier for the benchmark","string",NA,"expanded","asIs","Y","" +"geo_misclosureInfo","benchmarkEasting","Easting coordinate value of the benchmark","real","meter","expanded","*.###(round)","N","" +"geo_misclosureInfo","benchmarkHeight","Height of the benchmark","real","meter","expanded","*.###(round)","N","" +"geo_misclosureInfo","benchmarkNorthing","Northing coordinate value of the benchmark","real","meter","expanded","*.###(round)","N","" +"geo_misclosureInfo","closureID","Identifier for the closure mapped point","string",NA,"expanded","asIs","Y","" +"geo_misclosureInfo","closureEasting","Easting coordinate value of the closure","real","meter","expanded","*.###(round)","N","" +"geo_misclosureInfo","closureHeight","Height of the closure","real","meter","expanded","*.###(round)","N","" +"geo_misclosureInfo","closureNorthing","Northing coordinate value of the closure","real","meter","expanded","*.###(round)","N","" +"geo_misclosureInfo","misclosureEasting","Easting coordinate value of the misclosure","real","meter","expanded","*.###(round)","N","" +"geo_misclosureInfo","misclosureHeight","Height of the misclosure","real","meter","expanded","*.###(round)","N","" +"geo_misclosureInfo","misclosureNorthing","Northing coordinate value of the misclosure","real","meter","expanded","*.###(round)","N","" +"geo_misclosureInfo","dataQF","Data quality flag","string",NA,"expanded","asIs","N","" +"geo_missingLine","uid","Unique ID within NEON database; an identifier for the record","string",NA,"expanded","asIs","N","" +"geo_missingLine","domainID","Unique identifier of the NEON domain","string",NA,"expanded","asIs","N","" +"geo_missingLine","siteID","NEON site code","string",NA,"expanded","asIs","N","" +"geo_missingLine","namedLocation","Name of the measurement location in the NEON database","string",NA,"expanded","asIs","N","" +"geo_missingLine","startDate","The start date-time or interval during which an event occurred","dateTime",NA,"expanded","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_missingLine","surveyEndDate","The date-time indicating the end of all surveying activities","dateTime",NA,"expanded","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_missingLine","eventID","An identifier for the set of information associated with the event, which includes information about the place and time of the event","string",NA,"expanded","asIs","Y","" +"geo_missingLine","attemptNumber","The attempt number associated with the missing line workflow","integer","number","expanded","integer","Y","" +"geo_missingLine","missingLinePoint1","Identifier for the first point used in the missing line workflow","string",NA,"expanded","asIs","Y","" +"geo_missingLine","missingLinePoint2","Identifier for the second point used in the missing line workflow","string",NA,"expanded","asIs","Y","" +"geo_missingLine","missingLineResultsHD","The difference in horizontal distance calculated between the first and second shots during the missing line workflow","real","meter","expanded","*.###(round)","N","" +"geo_missingLine","missingLineResultsdH","The difference in elevation calculated between the first and second shots during the missing line workflow","real","meter","expanded","*.###(round)","N","" +"geo_missingLine","missingLineReultsStDev","The standard deviation calculated between the first and second shots during the missing line workflow","real","meter","expanded","*.###(round)","N","" +"geo_missingLine","missingLineResultsStationDeviation","The station deviation calculated between the first and second shots during the missing line workflow","real","meter","expanded","*.###(round)","N","" +"geo_missingLine","remarks","Technician notes; free text comments accompanying the record","string",NA,"expanded","asIs","N","" +"geo_missingLine","dataQF","Data quality flag","string",NA,"expanded","asIs","N","" +"geo_pebbleCount","uid","Unique ID within NEON database; an identifier for the record","string",NA,"expanded","asIs","N","" +"geo_pebbleCount","domainID","Unique identifier of the NEON domain","string",NA,"expanded","asIs","N","" +"geo_pebbleCount","siteID","NEON site code","string",NA,"expanded","asIs","N","" +"geo_pebbleCount","namedLocation","Name of the measurement location in the NEON database","string",NA,"expanded","asIs","N","" +"geo_pebbleCount","eventID","An identifier for the set of information associated with the event, which includes information about the place and time of the event","string",NA,"expanded","asIs","Y","" +"geo_pebbleCount","measurementLocation","Location of the technician for making a measurement","string",NA,"expanded","asIs","Y","" +"geo_pebbleCount","totalStationLocation","Identifier for sequential total station locations throughout the survey","string",NA,"expanded","asIs","N","" +"geo_pebbleCount","startDate","The start date-time or interval during which an event occurred","dateTime",NA,"expanded","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_pebbleCount","surveyEndDate","The date-time indicating the end of all surveying activities","dateTime",NA,"expanded","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_pebbleCount","habitatType","Habitat type sampled","string",NA,"expanded","LOV","N","geo.habitatType" +"geo_pebbleCount","pebbleCountNumber","A tallied record of samples measured during the pebble count survey","integer","number","expanded","integer","Y","" +"geo_pebbleCount","pebbleSize","The bin class associated with the substrate measured during the pebble count survey","string",NA,"expanded","LOV","N","geo.diameterBin" +"geo_pebbleCount","dataQF","Data quality flag","string",NA,"expanded","asIs","N","" +"geo_pebbleFieldData","uid","Unique ID within NEON database; an identifier for the record","string",NA,"basic","asIs","N","" +"geo_pebbleFieldData","protocolActivity","Protocol activity under which the event occurred","string",NA,"basic","LOV","N","geo.protocolActivity" +"geo_pebbleFieldData","domainID","Unique identifier of the NEON domain","string",NA,"basic","asIs","N","" +"geo_pebbleFieldData","siteID","NEON site code","string",NA,"basic","asIs","N","" +"geo_pebbleFieldData","namedLocation","Name of the measurement location in the NEON database","string",NA,"basic","asIs","N","" +"geo_pebbleFieldData","eventID","An identifier for the set of information associated with the event, which includes information about the place and time of the event","string",NA,"basic","asIs","Y","" +"geo_pebbleFieldData","geodeticDatum","Model used to measure horizontal position on the earth","string",NA,"basic","asIs","N","" +"geo_pebbleFieldData","startDate","The start date-time or interval during which an event occurred","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_pebbleFieldData","surveyEndDate","The date-time indicating the end of all surveying activities","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_pebbleFieldData","samplingProtocolVersion","The NEON document number and version where detailed information regarding the sampling method used is available; format NEON.DOC.######vX","string",NA,"basic","LOV","N","geo.samplingProtocolVersion" +"geo_pebbleFieldData","measuredBy","An identifier for the technician who measured or collected the data","string",NA,"basic","asIs","N","" +"geo_pebbleFieldData","recordedBy","An identifier for the technician who recorded the data","string",NA,"basic","asIs","N","" +"geo_pebbleFieldData","remarks","Technician notes; free text comments accompanying the record","string",NA,"basic","asIs","N","" +"geo_pebbleFieldData","pebbleCountD5","The particle size that 5 percent of the particles are equal to or smaller than in the pebble count distribution","string",NA,"basic","asIs","N","" +"geo_pebbleFieldData","pebbleCountD16","The particle size that 16 percent of the particles are equal to or smaller than in the pebble count distribution","string",NA,"basic","asIs","N","" +"geo_pebbleFieldData","pebbleCountD50","The particle size that 50 percent of the particles are equal to or smaller than in the pebble count distribution","string",NA,"basic","asIs","N","" +"geo_pebbleFieldData","pebbleCountD84","The particle size that 84 percent of the particles are equal to or smaller than in the pebble count distribution","string",NA,"basic","asIs","N","" +"geo_pebbleFieldData","dataQF","Data quality flag","string",NA,"basic","asIs","N","" +"geo_processedSurveyData","uid","Unique ID within NEON database; an identifier for the record","string",NA,"basic","asIs","N","" +"geo_processedSurveyData","domainID","Unique identifier of the NEON domain","string",NA,"basic","asIs","N","" +"geo_processedSurveyData","siteID","NEON site code","string",NA,"basic","asIs","N","" +"geo_processedSurveyData","namedLocation","Name of the measurement location in the NEON database","string",NA,"basic","asIs","N","" +"geo_processedSurveyData","startDate","The start date-time or interval during which an event occurred","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_processedSurveyData","surveyEndDate","The date-time indicating the end of all surveying activities","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_processedSurveyData","eventID","An identifier for the set of information associated with the event, which includes information about the place and time of the event","string",NA,"basic","asIs","Y","" +"geo_processedSurveyData","surveyPointID","Unique identifier for each mapped point collected during the survey","string",NA,"basic","asIs","Y","" +"geo_processedSurveyData","decimalLatitude","The geographic latitude (in decimal degrees, WGS84) of the geographic center of the reference area","real","decimalDegree","basic","*.######(round)","N","" +"geo_processedSurveyData","decimalLongitude","The geographic longitude (in decimal degrees, WGS84) of the geographic center of the reference area","real","decimalDegree","basic","*.######(round)","N","" +"geo_processedSurveyData","easting","Geographic coordinate specifying the east-west position of a point on the Earth's surface (Universal Transverse Mercator, UTM)","real","meter","basic","*.###(round)","N","" +"geo_processedSurveyData","northing","Geographic coordinate specifying the north-south position of a point on the Earth's surface (Universal Transverse Mercator, UTM)","real","meter","basic","*.###(round)","N","" +"geo_processedSurveyData","elevation","Elevation (in meters) above sea level","real","meter","basic","*.##(round)","N","" +"geo_processedSurveyData","utmZone","UTM zone","string",NA,"basic","LOV","N","Named Location TOS UTM Zones" +"geo_processedSurveyData","geodeticDatum","Model used to measure horizontal position on the earth","string",NA,"basic","LOV","N","Named Location TOS Geodetic Data" +"geo_processedSurveyData","mapCode","A general classification code used to categorize mapped points collected during the survey","string",NA,"basic","LOV","Y","geo.mapCode" +"geo_processedSurveyData","remarks","Technician notes; free text comments accompanying the record","string",NA,"basic","asIs","N","" +"geo_processedSurveyData","processedSurveyVersion","The version of the post-processing method that can be linked to the specific survey data package stored in the cloud","string",NA,"basic","LOV","N","geo.processedSurveyVersion" +"geo_processedSurveyData","dataQF","Data quality flag","string",NA,"basic","asIs","N","" +"geo_rapidHabitatAssessment","uid","Unique ID within NEON database; an identifier for the record","string",NA,"basic","asIs","N","" +"geo_rapidHabitatAssessment","domainID","Unique identifier of the NEON domain","string",NA,"basic","asIs","N","" +"geo_rapidHabitatAssessment","siteID","NEON site code","string",NA,"basic","asIs","N","" +"geo_rapidHabitatAssessment","namedLocation","Name of the measurement location in the NEON database","string",NA,"basic","asIs","N","" +"geo_rapidHabitatAssessment","startDate","The start date-time or interval during which an event occurred","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_rapidHabitatAssessment","surveyEndDate","The date-time indicating the end of all surveying activities","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_rapidHabitatAssessment","eventID","An identifier for the set of information associated with the event, which includes information about the place and time of the event","string",NA,"basic","asIs","Y","" +"geo_rapidHabitatAssessment","habitatType","Habitat type sampled","string",NA,"basic","LOV","Y","geo.usdsHabitatType" +"geo_rapidHabitatAssessment","habitatOrder","An ascending order count of habitat types mapped during the survey","integer",NA,"basic","integer","Y","" +"geo_rapidHabitatAssessment","habitatLength","The length of the stream habitat unit, measured along the thalweg","real","meter","basic","*.##(round)","N","" +"geo_rapidHabitatAssessment","wettedWidth","Width of the wetted stream at time of measurement","real","meter","basic","*.##(round)","N","" +"geo_rapidHabitatAssessment","maxDepth","Maximum depth","real","meter","basic","*.##(round)","N","" +"geo_rapidHabitatAssessment","percentCWD","A visual estimate of the percentage of course woody debris within the stream habitat unit","real","percent","basic","*.##(round)","N","" +"geo_rapidHabitatAssessment","substrateType","A categorical classification of substrate used to identify streambed types during the survey","string",NA,"basic","LOV","N","geo.substrateType" +"geo_rapidHabitatAssessment","percentRiparianCover","A visual estimate of the percentage of riparian cover above the stream habitat unit","real","percent","basic","*.##(round)","N","" +"geo_rapidHabitatAssessment","maxPDOP","The maximum value of the Position Dilution of Precision (PDOP), a unitless figure of merit expressing the relationship between the error in user position and the error in satellite position. A low DOP value indicates a higher probability of accuracy.","real",NA,"basic","*.#(round)","N","" +"geo_rapidHabitatAssessment","maxHDOP","The maximum value of the Horizontal Dilution of Precision (HDOP), a unitless figure of merit expressing the relationship between the error in user position and the error in satellite position. A low DOP value indicates a higher probability of accuracy.","real",NA,"basic","*.#(round)","N","" +"geo_rapidHabitatAssessment","gnssCorrectionType","Differential correction technique used to post process the Global Navigation Satellite System (GNSS) data","string",NA,"basic","LOV","N","geo.gnssCorrectionType" +"geo_rapidHabitatAssessment","gnssUsed","The Global Navigation Satellite System (GNSS) unit used to collect the associated latitude and longitude","string",NA,"basic","LOV","N","geo.gnssUsed" +"geo_rapidHabitatAssessment","elevation","Elevation (in meters) above sea level","real","meter","basic","*.##(round)","N","" +"geo_rapidHabitatAssessment","elevationUncertainty","Uncertainty in elevation values (in meters)","real","meter","basic","*.##(round)","N","" +"geo_rapidHabitatAssessment","coordinateUncertainty","The horizontal distance (in meters) from the given decimalLatitude and decimalLongitude describing the smallest circle containing the whole of the Location. Zero is not a valid value for this term","real","meter","basic","*.##(round)","N","" +"geo_rapidHabitatAssessment","gnssStDev","The spread of Global Navigation Satellite System (GNSS) positions averaged for the point","real","meter","basic","*.###(round)","N","" +"geo_rapidHabitatAssessment","decimalLatitude","The geographic latitude (in decimal degrees, WGS84) of the geographic center of the reference area","real","decimalDegree","basic","*.######(round)","N","" +"geo_rapidHabitatAssessment","decimalLongitude","The geographic longitude (in decimal degrees, WGS84) of the geographic center of the reference area","real","decimalDegree","basic","*.######(round)","N","" +"geo_rapidHabitatAssessment","easting","Geographic coordinate specifying the east-west position of a point on the Earth's surface (Universal Transverse Mercator, UTM)","real","meter","basic","*.###(round)","N","" +"geo_rapidHabitatAssessment","northing","Geographic coordinate specifying the north-south position of a point on the Earth's surface (Universal Transverse Mercator, UTM)","real","meter","basic","*.###(round)","N","" +"geo_rapidHabitatAssessment","utmZone","UTM zone","string",NA,"basic","LOV","N","Named Location TOS UTM Zones" +"geo_rapidHabitatAssessment","geodeticDatum","Model used to measure horizontal position on the earth","string",NA,"basic","LOV","N","Named Location TOS Geodetic Data" +"geo_rapidHabitatAssessment","processedSurveyVersion","The version of the post-processing method that can be linked to the specific survey data package stored in the cloud","string",NA,"basic","LOV","N","geo.processedSurveyVersion" +"geo_rapidHabitatAssessment","remarks","Technician notes; free text comments accompanying the record","string",NA,"basic","asIs","N","" +"geo_rapidHabitatAssessment","dataQF","Data quality flag","string",NA,"basic","asIs","N","" +"geo_rawSurveyData","uid","Unique ID within NEON database; an identifier for the record","string",NA,"expanded","asIs","N","" +"geo_rawSurveyData","domainID","Unique identifier of the NEON domain","string",NA,"expanded","asIs","N","" +"geo_rawSurveyData","siteID","NEON site code","string",NA,"expanded","asIs","N","" +"geo_rawSurveyData","namedLocation","Name of the measurement location in the NEON database","string",NA,"expanded","asIs","N","" +"geo_rawSurveyData","startDate","The start date-time or interval during which an event occurred","dateTime",NA,"expanded","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_rawSurveyData","surveyEndDate","The date-time indicating the end of all surveying activities","dateTime",NA,"expanded","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_rawSurveyData","eventID","An identifier for the set of information associated with the event, which includes information about the place and time of the event","string",NA,"expanded","asIs","Y","" +"geo_rawSurveyData","surveyPointID","Unique identifier for each mapped point collected during the survey","string",NA,"expanded","asIs","Y","" +"geo_rawSurveyData","relativeEasting","Coordinate specifying the east-west position of the mapped point measured eastward from the origin of the starting location of the survey","real","meter","expanded","*.######(round)","N","" +"geo_rawSurveyData","relativeNorthing","Coordinate specifying the north-south position of the mapped point measured northward from the origin of the starting location of the survey","real","meter","expanded","*.######(round)","N","" +"geo_rawSurveyData","relativeHeight","The vertical height of the mapped point measured relative to vertical height assigned to the permanent benchmarks","real","meter","expanded","*.######(round)","N","" +"geo_rawSurveyData","relativeHorizontalAngle","The horizontal angle of the mapped point measured relative to the origin of the permanent benchmarks","real","radian","expanded","*.######(round)","N","" +"geo_rawSurveyData","relativeVerticalAngle","The vertical angle of the mapped point measured relative to the origin of the permanent benchmarks","real","radian","expanded","*.######(round)","N","" +"geo_rawSurveyData","relativeHorizontalDistance","The horizontal distance between the head unit of the total station and the mapped point","real","meter","expanded","*.######(round)","N","" +"geo_rawSurveyData","heightOfRod","The height of the survey rod used to measure the mapped point","real","meter","expanded","*.###(round)","N","" +"geo_rawSurveyData","atmosphericCorrection","A value entered into the total station settings to correct for the effects of the Earth's atmosphere","real","partsPerMillion","expanded","*.###(round)","N","" +"geo_rawSurveyData","processedSurveyVersion","The version of the post-processing method that can be linked to the specific survey data package stored in the cloud","string",NA,"expanded","LOV","N","geo.processedSurveyVersion" +"geo_rawSurveyData","remarks","Technician notes; free text comments accompanying the record","string",NA,"expanded","asIs","N","" +"geo_rawSurveyData","dataQF","Data quality flag","string",NA,"expanded","asIs","N","" +"geo_surveyPoints","uid","Unique ID within NEON database; an identifier for the record","string",NA,"expanded","asIs","N","" +"geo_surveyPoints","domainID","Unique identifier of the NEON domain","string",NA,"expanded","asIs","N","" +"geo_surveyPoints","siteID","NEON site code","string",NA,"expanded","asIs","N","" +"geo_surveyPoints","namedLocation","Name of the measurement location in the NEON database","string",NA,"expanded","asIs","N","" +"geo_surveyPoints","startDate","The start date-time or interval during which an event occurred","dateTime",NA,"expanded","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_surveyPoints","surveyEndDate","The date-time indicating the end of all surveying activities","dateTime",NA,"expanded","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_surveyPoints","eventID","An identifier for the set of information associated with the event, which includes information about the place and time of the event","string",NA,"expanded","asIs","Y","" +"geo_surveyPoints","surveyPointID","Unique identifier for each mapped point collected during the survey","string",NA,"expanded","asIs","Y","" +"geo_surveyPoints","relativeEasting","Coordinate specifying the east-west position of the mapped point measured eastward from the origin of the starting location of the survey","real","meter","expanded","*.######(round)","N","" +"geo_surveyPoints","relativeNorthing","Coordinate specifying the north-south position of the mapped point measured northward from the origin of the starting location of the survey","real","meter","expanded","*.######(round)","N","" +"geo_surveyPoints","relativeHeight","The vertical height of the mapped point measured relative to vertical height assigned to the permanent benchmarks","real","meter","expanded","*.######(round)","N","" +"geo_surveyPoints","mapCode","A general classification code used to categorize mapped points collected during the survey","string",NA,"expanded","LOV","Y","geo.mapCode" +"geo_surveyPoints","processedSurveyVersion","The version of the post-processing method that can be linked to the specific survey data package stored in the cloud","string",NA,"expanded","LOV","N","geo.processedSurveyVersion" +"geo_surveyPoints","remarks","Technician notes; free text comments accompanying the record","string",NA,"expanded","asIs","N","" +"geo_surveyPoints","dataQF","Data quality flag","string",NA,"expanded","asIs","N","" +"geo_surveySummary","uid","Unique ID within NEON database; an identifier for the record","string",NA,"basic","asIs","N","" +"geo_surveySummary","domainID","Unique identifier of the NEON domain","string",NA,"basic","asIs","N","" +"geo_surveySummary","siteID","NEON site code","string",NA,"basic","asIs","N","" +"geo_surveySummary","namedLocation","Name of the measurement location in the NEON database","string",NA,"basic","asIs","N","" +"geo_surveySummary","startDate","The start date-time or interval during which an event occurred","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_surveySummary","surveyEndDate","The date-time indicating the end of all surveying activities","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_surveySummary","samplingImpractical","Samples and/or measurements were not collected due to the indicated circumstance","string",NA,"basic","LOV","N","samplingImpractical" +"geo_surveySummary","eventID","An identifier for the set of information associated with the event, which includes information about the place and time of the event","string",NA,"basic","asIs","Y","" +"geo_surveySummary","surveyBoutTypeID","Identifier for the type of survey performed","string",NA,"basic","LOV","N","geo.protocolActivity" +"geo_surveySummary","meanBankfullWidth","The average bankfull width of all channel cross-sections mapped during the survey","real","meter","basic","*.##(round)","N","" +"geo_surveySummary","S1habitatID","Unique identifier for habitat type of sensor set 1 mapped during the survey","string",NA,"basic","asIs","N","" +"geo_surveySummary","S2habitatID","Unique identifier for habitat type of sensor set 2 mapped during the survey","string",NA,"basic","asIs","N","" +"geo_surveySummary","sensorSetLength","The distance along the mapped thalweg between sensor set 1 and sensor set 2","real","meter","basic","*.##(round)","N","" +"geo_surveySummary","sensorSetSlope","The slope of the thalweg between sensor set 1 and sensor set 2","real","percent","basic","*.##(round)","N","" +"geo_surveySummary","sensorSetElevDiff","The difference in thalweg elevation between sensor set 1 and sensor set 2","real","meter","basic","*.##(round)","N","" +"geo_surveySummary","thalwegLength","The total thalweg length mapped during the survey","real","meter","basic","*.##(round)","N","" +"geo_surveySummary","thalwegSlope","The slope of the thalweg length mapped during the survey","real","percent","basic","*.##(round)","N","" +"geo_surveySummary","channelSinuosityIndex","The total thalweg length divided by the length of the line that comprises the shortest distance between the most upstream and downstream mapped points","real",NA,"basic","*.##(round)","N","" +"geo_surveySummary","percentDry","The percentage of thalweg length that contained a dry channel","real","percent","basic","*.##(round)","N","" +"geo_surveySummary","percentComplete","The percentage of the total NEON monitoring reach mapped during the survey","real","percent","basic","*.##(round)","N","" +"geo_surveySummary","percentSubterranean","The percentage of the survey in which sub-surface flow was mapped","real","percent","basic","*.##(round)","N","" +"geo_surveySummary","totalTransformationError","A root mean square error, calculated based on residual errors, to indicate the general suitability of the derived geospatial transformation","real",NA,"basic","*.##(round)","N","" +"geo_surveySummary","samplingProtocolVersion","The NEON document number and version where detailed information regarding the sampling method used is available; format NEON.DOC.######vX","string",NA,"basic","LOV","N","geo.samplingProtocolVersion" +"geo_surveySummary","sopVersion","Version of the SOP used to create the final data","string",NA,"basic","asIs","N","" +"geo_surveySummary","dataFileName","Name of file or folder containing data, including file extension","string",NA,"basic","asIs","N","" +"geo_surveySummary","dataFilePath","The system path identifying the data file location","uri",NA,"basic","asIs","N","" +"geo_surveySummary","rawDataFileName","Name of file or folder containing raw data, including file extension","string",NA,"basic","asIs","N","" +"geo_surveySummary","rawDataFilePath","The system path identifying the raw data file location","uri",NA,"basic","asIs","N","" +"geo_surveySummary","remarks","Technician notes; free text comments accompanying the record","string",NA,"basic","asIs","N","" +"geo_surveySummary","processedSurveyVersion","The version of the post-processing method that can be linked to the specific survey data package stored in the cloud","string",NA,"basic","LOV","N","geo.processedSurveyVersion" +"geo_surveySummary","dataQF","Data quality flag","string",NA,"basic","asIs","N","" +"geo_surveySummary","publicationDate","Date of data publication on the NEON data portal","dateTime",NA,"appended by stackByTable",NA,"N","" +"geo_surveySummary","release","Identifier for data release","string",NA,"appended by stackByTable",NA,"N","" +"geo_thalwegByHabitatID","uid","Unique ID within NEON database; an identifier for the record","string",NA,"basic","asIs","N","" +"geo_thalwegByHabitatID","domainID","Unique identifier of the NEON domain","string",NA,"basic","asIs","N","" +"geo_thalwegByHabitatID","siteID","NEON site code","string",NA,"basic","asIs","N","" +"geo_thalwegByHabitatID","namedLocation","Name of the measurement location in the NEON database","string",NA,"basic","asIs","N","" +"geo_thalwegByHabitatID","startDate","The start date-time or interval during which an event occurred","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_thalwegByHabitatID","surveyEndDate","The date-time indicating the end of all surveying activities","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_thalwegByHabitatID","eventID","An identifier for the set of information associated with the event, which includes information about the place and time of the event","string",NA,"basic","asIs","Y","" +"geo_thalwegByHabitatID","habitatID","Unique identifier for each habit type mapped during the survey","string",NA,"basic","asIs","Y","" +"geo_thalwegByHabitatID","habitatLength","The length of the stream habitat unit, measured along the thalweg","real","meter","basic","*.##(round)","N","" +"geo_thalwegByHabitatID","habitatArea","The area of the stream habitat unit, measured within the wetted edges of the channel","real","meter","basic","*.##(round)","N","" +"geo_thalwegByHabitatID","percentTotalThalwegLength","The percentage of the total thalweg length that the stream habitat unit comprises","real","percent","basic","*.##(round)","N","" +"geo_thalwegByHabitatID","percentSubterranean","The percentage of the survey in which sub-surface flow was mapped","real","percent","basic","*.##(round)","N","" +"geo_thalwegByHabitatID","percentDry","The percentage of thalweg length that contained a dry channel","real","percent","basic","*.##(round)","N","" +"geo_thalwegByHabitatID","remarks","Technician notes; free text comments accompanying the record","string",NA,"basic","asIs","N","" +"geo_thalwegByHabitatID","processedSurveyVersion","The version of the post-processing method that can be linked to the specific survey data package stored in the cloud","string",NA,"basic","LOV","N","geo.processedSurveyVersion" +"geo_thalwegByHabitatID","dataQF","Data quality flag","string",NA,"basic","asIs","N","" +"geo_thalwegByHabitatType","uid","Unique ID within NEON database; an identifier for the record","string",NA,"basic","asIs","N","" +"geo_thalwegByHabitatType","domainID","Unique identifier of the NEON domain","string",NA,"basic","asIs","N","" +"geo_thalwegByHabitatType","siteID","NEON site code","string",NA,"basic","asIs","N","" +"geo_thalwegByHabitatType","namedLocation","Name of the measurement location in the NEON database","string",NA,"basic","asIs","N","" +"geo_thalwegByHabitatType","startDate","The start date-time or interval during which an event occurred","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_thalwegByHabitatType","surveyEndDate","The date-time indicating the end of all surveying activities","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_thalwegByHabitatType","eventID","An identifier for the set of information associated with the event, which includes information about the place and time of the event","string",NA,"basic","asIs","Y","" +"geo_thalwegByHabitatType","habitatType","Habitat type sampled","string",NA,"basic","LOV","Y","geo.usdsHabitatType" +"geo_thalwegByHabitatType","habitatLength","The length of the stream habitat unit, measured along the thalweg","real","meter","basic","*.##(round)","N","" +"geo_thalwegByHabitatType","habitatArea","The area of the stream habitat unit, measured within the wetted edges of the channel","real","meter","basic","*.##(round)","N","" +"geo_thalwegByHabitatType","percentTotalThalwegLength","The percentage of the total thalweg length that the stream habitat unit comprises","real","percent","basic","*.##(round)","N","" +"geo_thalwegByHabitatType","remarks","Technician notes; free text comments accompanying the record","string",NA,"basic","asIs","N","" +"geo_thalwegByHabitatType","processedSurveyVersion","The version of the post-processing method that can be linked to the specific survey data package stored in the cloud","string",NA,"basic","LOV","N","geo.processedSurveyVersion" +"geo_thalwegByHabitatType","dataQF","Data quality flag","string",NA,"basic","asIs","N","" +"geo_thalwegLongProfile","uid","Unique ID within NEON database; an identifier for the record","string",NA,"basic","asIs","N","" +"geo_thalwegLongProfile","domainID","Unique identifier of the NEON domain","string",NA,"basic","asIs","N","" +"geo_thalwegLongProfile","siteID","NEON site code","string",NA,"basic","asIs","N","" +"geo_thalwegLongProfile","namedLocation","Name of the measurement location in the NEON database","string",NA,"basic","asIs","N","" +"geo_thalwegLongProfile","startDate","The start date-time or interval during which an event occurred","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_thalwegLongProfile","surveyEndDate","The date-time indicating the end of all surveying activities","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_thalwegLongProfile","eventID","An identifier for the set of information associated with the event, which includes information about the place and time of the event","string",NA,"basic","asIs","Y","" +"geo_thalwegLongProfile","surveyPointID","Unique identifier for each mapped point collected during the survey","string",NA,"basic","asIs","Y","" +"geo_thalwegLongProfile","decimalLatitude","The geographic latitude (in decimal degrees, WGS84) of the geographic center of the reference area","real","decimalDegree","basic","*.######(round)","N","" +"geo_thalwegLongProfile","decimalLongitude","The geographic longitude (in decimal degrees, WGS84) of the geographic center of the reference area","real","decimalDegree","basic","*.######(round)","N","" +"geo_thalwegLongProfile","mapCode","A general classification code used to categorize mapped points collected during the survey","string",NA,"basic","LOV","Y","geo.mapCode" +"geo_thalwegLongProfile","easting","Geographic coordinate specifying the east-west position of a point on the Earth's surface (Universal Transverse Mercator, UTM)","real","meter","basic","*.###(round)","N","" +"geo_thalwegLongProfile","northing","Geographic coordinate specifying the north-south position of a point on the Earth's surface (Universal Transverse Mercator, UTM)","real","meter","basic","*.###(round)","N","" +"geo_thalwegLongProfile","elevation","Elevation (in meters) above sea level","real","meter","basic","*.##(round)","N","" +"geo_thalwegLongProfile","utmZone","UTM zone","string",NA,"basic","LOV","N","Named Location TOS UTM Zones" +"geo_thalwegLongProfile","geodeticDatum","Model used to measure horizontal position on the earth","string",NA,"basic","LOV","N","Named Location TOS Geodetic Data" +"geo_thalwegLongProfile","processedSurveyVersion","The version of the post-processing method that can be linked to the specific survey data package stored in the cloud","string",NA,"basic","LOV","N","geo.processedSurveyVersion" +"geo_thalwegLongProfile","remarks","Technician notes; free text comments accompanying the record","string",NA,"basic","asIs","N","" +"geo_thalwegLongProfile","dataQF","Data quality flag","string",NA,"basic","asIs","N","" +"geo_totalStation","uid","Unique ID within NEON database; an identifier for the record","string",NA,"expanded","asIs","N","" +"geo_totalStation","domainID","Unique identifier of the NEON domain","string",NA,"expanded","asIs","N","" +"geo_totalStation","siteID","NEON site code","string",NA,"expanded","asIs","N","" +"geo_totalStation","namedLocation","Name of the measurement location in the NEON database","string",NA,"expanded","asIs","N","" +"geo_totalStation","startDate","The start date-time or interval during which an event occurred","dateTime",NA,"expanded","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_totalStation","surveyEndDate","The date-time indicating the end of all surveying activities","dateTime",NA,"expanded","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_totalStation","eventID","An identifier for the set of information associated with the event, which includes information about the place and time of the event","string",NA,"expanded","asIs","Y","" +"geo_totalStation","totalStationLocation","Identifier for sequential total station locations throughout the survey","string",NA,"expanded","asIs","Y","" +"geo_totalStation","firstControlPointType","The first type of control point used to orient the total station at a given location","string",NA,"expanded","LOV","N","geo.pointType" +"geo_totalStation","firstControlPointID","Identifier for the first control point used to orient the total station at a given location","string",NA,"expanded","asIs","Y","" +"geo_totalStation","secondControlPointType","The second type of control point used to orient the total station at a given location","string",NA,"expanded","LOV","N","geo.pointType" +"geo_totalStation","secondControlPointID","Identifier for the second control point used to orient the total station at a given location","string",NA,"expanded","asIs","Y","" +"geo_totalStation","thirdControlPointType","The third type of control point used to orient the total station at a given location","string",NA,"expanded","LOV","N","geo.pointType" +"geo_totalStation","thirdControlPointID","Identifier for the third control point used to orient the total station at a given location","string",NA,"expanded","asIs","Y","" +"geo_totalStation","stationDeviationDevH","The station deviation of the height calculated when two or three control points are used for orientation","real","meter","expanded","*.###(round)","N","" +"geo_totalStation","stationDeviationHADegrees","The station deviation of the horizontal angle in degrees calculated when three control points are used for orientation","real","degree","expanded","*.##(round)","N","" +"geo_totalStation","stationDeviationHAMinutes","The station deviation of the horizontal angle in minutes calculated when three control points are used for orientation","real","arcMinute","expanded","*.##(round)","N","" +"geo_totalStation","stationDeviationHASeconds","The station deviation of the horizontal angle in seconds calculated when three control points are used for orientation","real","arcSecond","expanded","*.##(round)","N","" +"geo_totalStation","stationDeviationPos","The station deviation of the position of the total station robotic head calculated when three control points are used for orientation","real","meter","expanded","*.###(round)","N","" +"geo_totalStation","firstThalwegPoint","Identifier for the first thalweg point mapped from a given total station location","integer","number","expanded","integer","N","" +"geo_totalStation","firstThalwegPointName","Name of the first thalweg point mapped from a given total station location","string",NA,"expanded","asIs","N","" +"geo_totalStation","lastThalwegPoint","Identifier for the last thalweg point mapped from a given total station location","integer","number","expanded","integer","N","" +"geo_totalStation","lastThalwegPointName","Name of the last thalweg point mapped from a given total station location","string",NA,"expanded","asIs","N","" +"geo_totalStation","featuresSurveyed","List of geomorphic features that were surveyed from a given total station location","string",NA,"expanded","asIs","N","" +"geo_totalStation","featuresSurveyedOther","List of additional geomorphic features that were surveyed from a given total station location","string",NA,"expanded","asIs","N","" +"geo_totalStation","remarks","Technician notes; free text comments accompanying the record","string",NA,"expanded","asIs","N","" +"geo_totalStation","dataQF","Data quality flag","string",NA,"expanded","asIs","N","" +"geo_transectBankfullWidths","uid","Unique ID within NEON database; an identifier for the record","string",NA,"basic","asIs","N","" +"geo_transectBankfullWidths","domainID","Unique identifier of the NEON domain","string",NA,"basic","asIs","N","" +"geo_transectBankfullWidths","siteID","NEON site code","string",NA,"basic","asIs","N","" +"geo_transectBankfullWidths","namedLocation","Name of the measurement location in the NEON database","string",NA,"basic","asIs","N","" +"geo_transectBankfullWidths","startDate","The start date-time or interval during which an event occurred","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_transectBankfullWidths","surveyEndDate","The date-time indicating the end of all surveying activities","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_transectBankfullWidths","eventID","An identifier for the set of information associated with the event, which includes information about the place and time of the event","string",NA,"basic","asIs","Y","" +"geo_transectBankfullWidths","transectID","An identifier for the transect","string",NA,"basic","LOV","Y","geo.transectID" +"geo_transectBankfullWidths","bankfullWidth","The measured bankfull width of the cross-section transect","real","meter","basic","*.##(round)","N","" +"geo_transectBankfullWidths","processedSurveyVersion","The version of the post-processing method that can be linked to the specific survey data package stored in the cloud","string",NA,"basic","LOV","N","geo.processedSurveyVersion" +"geo_transectBankfullWidths","remarks","Technician notes; free text comments accompanying the record","string",NA,"basic","asIs","N","" +"geo_transectBankfullWidths","dataQF","Data quality flag","string",NA,"basic","asIs","N","" +"geo_trimbleData","uid","Unique ID within NEON database; an identifier for the record","string",NA,"expanded","asIs","N","" +"geo_trimbleData","domainID","Unique identifier of the NEON domain","string",NA,"expanded","asIs","N","" +"geo_trimbleData","siteID","NEON site code","string",NA,"expanded","asIs","N","" +"geo_trimbleData","namedLocation","Name of the measurement location in the NEON database","string",NA,"expanded","asIs","N","" +"geo_trimbleData","startDate","The start date-time or interval during which an event occurred","dateTime",NA,"expanded","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_trimbleData","surveyEndDate","The date-time indicating the end of all surveying activities","dateTime",NA,"expanded","yyyy-MM-dd'T'HH:mm'Z'(floor)","N","" +"geo_trimbleData","eventID","An identifier for the set of information associated with the event, which includes information about the place and time of the event","string",NA,"expanded","asIs","Y","" +"geo_trimbleData","surveyPointID","Unique identifier for each mapped point collected during the survey","string",NA,"expanded","asIs","Y","" +"geo_trimbleData","maxPDOP","The maximum value of the Position Dilution of Precision (PDOP), a unitless figure of merit expressing the relationship between the error in user position and the error in satellite position. A low DOP value indicates a higher probability of accuracy.","real",NA,"expanded","*.#(round)","N","" +"geo_trimbleData","maxHDOP","The maximum value of the Horizontal Dilution of Precision (HDOP), a unitless figure of merit expressing the relationship between the error in user position and the error in satellite position. A low DOP value indicates a higher probability of accuracy.","real",NA,"expanded","*.#(round)","N","" +"geo_trimbleData","gnssCorrectionType","Differential correction technique used to post process the Global Navigation Satellite System (GNSS) data","string",NA,"expanded","LOV","N","geo.gnssCorrectionType" +"geo_trimbleData","gnssUsed","The Global Navigation Satellite System (GNSS) unit used to collect the associated latitude and longitude","string",NA,"expanded","LOV","N","geo.gnssUsed" +"geo_trimbleData","gnssDatafile","Name assigned to the Global Navigation Satellite System (GNSS) file","string",NA,"expanded","asIs","N","" +"geo_trimbleData","elevation","Elevation (in meters) above sea level","real","meter","expanded","*.##(round)","N","" +"geo_trimbleData","elevationUncertainty","Uncertainty in elevation values (in meters)","real","meter","expanded","*.##(round)","N","" +"geo_trimbleData","coordinateUncertainty","The horizontal distance (in meters) from the given decimalLatitude and decimalLongitude describing the smallest circle containing the whole of the Location. Zero is not a valid value for this term","real","meter","expanded","*.##(round)","N","" +"geo_trimbleData","gnssStDev","The spread of Global Navigation Satellite System (GNSS) positions averaged for the point","real","meter","expanded","*.###(round)","N","" +"geo_trimbleData","decimalLatitude","The geographic latitude (in decimal degrees, WGS84) of the geographic center of the reference area","real","decimalDegree","expanded","*.######(round)","N","" +"geo_trimbleData","decimalLongitude","The geographic longitude (in decimal degrees, WGS84) of the geographic center of the reference area","real","decimalDegree","expanded","*.######(round)","N","" +"geo_trimbleData","easting","Geographic coordinate specifying the east-west position of a point on the Earth's surface (Universal Transverse Mercator, UTM)","real","meter","expanded","*.###(round)","N","" +"geo_trimbleData","northing","Geographic coordinate specifying the north-south position of a point on the Earth's surface (Universal Transverse Mercator, UTM)","real","meter","expanded","*.###(round)","N","" +"geo_trimbleData","utmZone","UTM zone","string",NA,"expanded","LOV","N","Named Location TOS UTM Zones" +"geo_trimbleData","geodeticDatum","Model used to measure horizontal position on the earth","string",NA,"expanded","LOV","N","Named Location TOS Geodetic Data" +"geo_trimbleData","processedSurveyVersion","The version of the post-processing method that can be linked to the specific survey data package stored in the cloud","string",NA,"expanded","LOV","N","geo.processedSurveyVersion" +"geo_trimbleData","remarks","Technician notes; free text comments accompanying the record","string",NA,"expanded","asIs","N","" +"geo_trimbleData","dataQF","Data quality flag","string",NA,"expanded","asIs","N","" diff --git a/testdata/NEON_uri_testdata/10017_DELA_202306_RELEASE2025/dhp_perimagefile.csv b/testdata/NEON_uri_testdata/10017_DELA_202306_RELEASE2025/dhp_perimagefile.csv new file mode 100644 index 0000000..dfba5bd --- /dev/null +++ b/testdata/NEON_uri_testdata/10017_DELA_202306_RELEASE2025/dhp_perimagefile.csv @@ -0,0 +1,37 @@ +"uid","namedLocation","domainID","siteID","plotID","pointID","startDate","endDate","sampleID","subsampleID","cameraOrientation","cameraPosition","biophysicalCriteria","imageType","imageFileName","imageFileUrl","remarks","publicationDate","release" +"656b0080-9b89-4cad-b673-23abea4feecc","DELA_053.basePlot.dhp","D08","DELA","DELA_053","E10",2023-06-06 10:25:00,2023-06-06 13:39:00,"DHP.2023.23.DELA053","DHP.2023.23.DELA053.E10.UNDERSTORY","downward","extendedMonopole","OK - no known exceptions","understory","D08_7589.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_053/understory/D08_7589.NEF",NA,"20241118T132242Z","RELEASE-2025" +"416c66fc-392b-4d73-a347-cf526f3f5cdc","DELA_053.basePlot.dhp","D08","DELA","DELA_053","W6",2023-06-06 10:25:00,2023-06-06 13:39:00,"DHP.2023.23.DELA053","DHP.2023.23.DELA053.W6.UNDERSTORY","downward","extendedMonopole","OK - no known exceptions","understory","D08_7598.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_053/understory/D08_7598.NEF",NA,"20241118T132242Z","RELEASE-2025" +"ae51111e-baac-4e63-83f5-3c47f00be343","DELA_053.basePlot.dhp","D08","DELA","DELA_053","S6",2023-06-06 10:25:00,2023-06-06 13:39:00,"DHP.2023.23.DELA053","DHP.2023.23.DELA053.S6.UNDERSTORY","downward","extendedMonopole","OK - no known exceptions","understory","D08_7592.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_053/understory/D08_7592.NEF",NA,"20241118T132242Z","RELEASE-2025" +"40a16829-ad5a-4b07-822f-b7af6d904c43","DELA_053.basePlot.dhp","D08","DELA","DELA_053","E2",2023-06-06 10:25:00,2023-06-06 13:39:00,"DHP.2023.23.DELA053","DHP.2023.23.DELA053.E2.UNDERSTORY","downward","extendedMonopole","OK - no known exceptions","understory","D08_7585.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_053/understory/D08_7585.NEF",NA,"20241118T132242Z","RELEASE-2025" +"c1ac3243-22c6-49d7-8371-d0aaf90f15e2","DELA_053.basePlot.dhp","D08","DELA","DELA_053","N2",2023-06-06 10:25:00,2023-06-06 13:39:00,"DHP.2023.23.DELA053","DHP.2023.23.DELA053.N2.UNDERSTORY","downward","extendedMonopole","OK - no known exceptions","understory","D08_7580.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_053/understory/D08_7580.NEF",NA,"20241118T132242Z","RELEASE-2025" +"5300150f-3ecc-413e-a7d7-f40b67919917","DELA_053.basePlot.dhp","D08","DELA","DELA_053","W2",2023-06-06 10:25:00,2023-06-06 13:39:00,"DHP.2023.23.DELA053","DHP.2023.23.DELA053.W2.UNDERSTORY","downward","extendedMonopole","OK - no known exceptions","understory","D08_7595.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_053/understory/D08_7595.NEF",NA,"20241118T132242Z","RELEASE-2025" +"b148ad83-b55e-435f-95c1-1b6103e30802","DELA_053.basePlot.dhp","D08","DELA","DELA_053","W10",2023-06-06 10:25:00,2023-06-06 13:39:00,"DHP.2023.23.DELA053","DHP.2023.23.DELA053.W10.UNDERSTORY","downward","extendedMonopole","OK - no known exceptions","understory","D08_7601.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_053/understory/D08_7601.NEF",NA,"20241118T132242Z","RELEASE-2025" +"5eae8cab-0774-4c73-8739-e8496aaf11ca","DELA_053.basePlot.dhp","D08","DELA","DELA_053","N6",2023-06-06 10:25:00,2023-06-06 13:39:00,"DHP.2023.23.DELA053","DHP.2023.23.DELA053.N6.UNDERSTORY","downward","extendedMonopole","OK - no known exceptions","understory","D08_7582.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_053/understory/D08_7582.NEF",NA,"20241118T132242Z","RELEASE-2025" +"2ea8aa83-bae9-4479-99d2-6fd78134d23b","DELA_053.basePlot.dhp","D08","DELA","DELA_053","N10",2023-06-06 10:25:00,2023-06-06 13:39:00,"DHP.2023.23.DELA053","DHP.2023.23.DELA053.N10.UNDERSTORY","downward","extendedMonopole","OK - no known exceptions","understory","D08_7602.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_053/understory/D08_7602.NEF",NA,"20241118T132242Z","RELEASE-2025" +"e097e159-090a-4d84-8e0c-b65559765543","DELA_053.basePlot.dhp","D08","DELA","DELA_053","E6",2023-06-06 10:25:00,2023-06-06 13:39:00,"DHP.2023.23.DELA053","DHP.2023.23.DELA053.E6.UNDERSTORY","downward","extendedMonopole","OK - no known exceptions","understory","D08_7587.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_053/understory/D08_7587.NEF",NA,"20241118T132242Z","RELEASE-2025" +"350d1cba-8dc5-466d-a0f4-bf8192436f78","DELA_053.basePlot.dhp","D08","DELA","DELA_053","S10",2023-06-06 10:25:00,2023-06-06 13:39:00,"DHP.2023.23.DELA053","DHP.2023.23.DELA053.S10.UNDERSTORY","downward","extendedMonopole","OK - no known exceptions","understory","D08_7593.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_053/understory/D08_7593.NEF",NA,"20241118T132242Z","RELEASE-2025" +"6df95b55-5360-4c1c-9a17-1e9a1af8d251","DELA_053.basePlot.dhp","D08","DELA","DELA_053","S2",2023-06-06 10:25:00,2023-06-06 13:39:00,"DHP.2023.23.DELA053","DHP.2023.23.DELA053.S2.UNDERSTORY","downward","extendedMonopole","OK - no known exceptions","understory","D08_7591.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_053/understory/D08_7591.NEF",NA,"20241118T132242Z","RELEASE-2025" +"a20aad1c-6761-4891-91e6-324de1a3c5f9","DELA_046.basePlot.dhp","D08","DELA","DELA_046","S10",2023-06-06 11:15:00,2023-06-06 13:40:00,"DHP.2023.23.DELA046","DHP.2023.23.DELA046.S10.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7615.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_046/understory/D08_7615.NEF",NA,"20241118T132242Z","RELEASE-2025" +"cff0f05b-165f-4536-b719-78241bae9808","DELA_046.basePlot.dhp","D08","DELA","DELA_046","N2",2023-06-06 11:15:00,2023-06-06 13:40:00,"DHP.2023.23.DELA046","DHP.2023.23.DELA046.N2.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7604.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_046/understory/D08_7604.NEF",NA,"20241118T132242Z","RELEASE-2025" +"4067385d-094f-457b-987d-163501892a84","DELA_046.basePlot.dhp","D08","DELA","DELA_046","W6",2023-06-06 11:15:00,2023-06-06 13:40:00,"DHP.2023.23.DELA046","DHP.2023.23.DELA046.W6.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7617.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_046/understory/D08_7617.NEF",NA,"20241118T132242Z","RELEASE-2025" +"6aa36ae9-5e13-408d-a23e-6b95e433038d","DELA_046.basePlot.dhp","D08","DELA","DELA_046","N10",2023-06-06 11:15:00,2023-06-06 13:40:00,"DHP.2023.23.DELA046","DHP.2023.23.DELA046.N10.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7608.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_046/understory/D08_7608.NEF",NA,"20241118T132242Z","RELEASE-2025" +"0c1c0452-227c-41d8-b89b-e41f34d40936","DELA_046.basePlot.dhp","D08","DELA","DELA_046","E2",2023-06-06 11:15:00,2023-06-06 13:40:00,"DHP.2023.23.DELA046","DHP.2023.23.DELA046.E2.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7609.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_046/understory/D08_7609.NEF",NA,"20241118T132242Z","RELEASE-2025" +"243daa22-99c8-49bd-bde9-4f99e87fd437","DELA_046.basePlot.dhp","D08","DELA","DELA_046","S6",2023-06-06 11:15:00,2023-06-06 13:40:00,"DHP.2023.23.DELA046","DHP.2023.23.DELA046.S6.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7614.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_046/understory/D08_7614.NEF",NA,"20241118T132242Z","RELEASE-2025" +"4b4d8013-786d-45f0-a0b0-8c82045a2a1e","DELA_046.basePlot.dhp","D08","DELA","DELA_046","W10",2023-06-06 11:15:00,2023-06-06 13:40:00,"DHP.2023.23.DELA046","DHP.2023.23.DELA046.W10.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7618.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_046/understory/D08_7618.NEF",NA,"20241118T132242Z","RELEASE-2025" +"0f9ecc2a-61d8-4815-aa1d-e40ecfac193d","DELA_046.basePlot.dhp","D08","DELA","DELA_046","E10",2023-06-06 11:15:00,2023-06-06 13:40:00,"DHP.2023.23.DELA046","DHP.2023.23.DELA046.E10.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7612.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_046/understory/D08_7612.NEF",NA,"20241118T132242Z","RELEASE-2025" +"e498ce25-8c51-4d7e-893a-206d7c91b2e3","DELA_046.basePlot.dhp","D08","DELA","DELA_046","E6",2023-06-06 11:15:00,2023-06-06 13:40:00,"DHP.2023.23.DELA046","DHP.2023.23.DELA046.E6.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7611.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_046/understory/D08_7611.NEF",NA,"20241118T132242Z","RELEASE-2025" +"26ba2f87-542d-40fc-954b-65a4eeedf2af","DELA_046.basePlot.dhp","D08","DELA","DELA_046","S2",2023-06-06 11:15:00,2023-06-06 13:40:00,"DHP.2023.23.DELA046","DHP.2023.23.DELA046.S2.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7613.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_046/understory/D08_7613.NEF",NA,"20241118T132242Z","RELEASE-2025" +"5e2701f0-87f1-4903-85a6-cb790f76814a","DELA_046.basePlot.dhp","D08","DELA","DELA_046","N6",2023-06-06 11:15:00,2023-06-06 13:40:00,"DHP.2023.23.DELA046","DHP.2023.23.DELA046.N6.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7605.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_046/understory/D08_7605.NEF",NA,"20241118T132242Z","RELEASE-2025" +"a3c4b2c0-5500-484d-a820-f4a6201c9b58","DELA_046.basePlot.dhp","D08","DELA","DELA_046","W2",2023-06-06 11:15:00,2023-06-06 13:40:00,"DHP.2023.23.DELA046","DHP.2023.23.DELA046.W2.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7616.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_046/understory/D08_7616.NEF",NA,"20241118T132242Z","RELEASE-2025" +"1a4e3c18-2d94-4dd9-9822-cce427dff02b","DELA_051.basePlot.dhp","D08","DELA","DELA_051","S2",2023-06-06 11:28:00,2023-06-06 13:39:00,"DHP.2023.23.DELA051","DHP.2023.23.DELA051.S2.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7628.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_051/understory/D08_7628.NEF",NA,"20241118T132242Z","RELEASE-2025" +"0fa4b47e-284f-455b-bc27-771270df75f6","DELA_051.basePlot.dhp","D08","DELA","DELA_051","N2",2023-06-06 11:28:00,2023-06-06 13:39:00,"DHP.2023.23.DELA051","DHP.2023.23.DELA051.N2.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7621.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_051/understory/D08_7621.NEF",NA,"20241118T132242Z","RELEASE-2025" +"e3b65f67-4db5-4e85-bf99-d7894a177b8f","DELA_051.basePlot.dhp","D08","DELA","DELA_051","S10",2023-06-06 11:28:00,2023-06-06 13:39:00,"DHP.2023.23.DELA051","DHP.2023.23.DELA051.S10.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7630.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_051/understory/D08_7630.NEF",NA,"20241118T132242Z","RELEASE-2025" +"4e4e563f-553d-4939-b6cc-ba8d48b19c4b","DELA_051.basePlot.dhp","D08","DELA","DELA_051","E6",2023-06-06 11:28:00,2023-06-06 13:39:00,"DHP.2023.23.DELA051","DHP.2023.23.DELA051.E6.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7626.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_051/understory/D08_7626.NEF",NA,"20241118T132242Z","RELEASE-2025" +"0baf0d97-4f35-422d-9cd8-c227eb5a3f7a","DELA_051.basePlot.dhp","D08","DELA","DELA_051","W10",2023-06-06 11:28:00,2023-06-06 13:39:00,"DHP.2023.23.DELA051","DHP.2023.23.DELA051.W10.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7634.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_051/understory/D08_7634.NEF",NA,"20241118T132242Z","RELEASE-2025" +"628c6dda-2b12-4fc6-9db4-90857cf05855","DELA_051.basePlot.dhp","D08","DELA","DELA_051","N6",2023-06-06 11:28:00,2023-06-06 13:39:00,"DHP.2023.23.DELA051","DHP.2023.23.DELA051.N6.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7622.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_051/understory/D08_7622.NEF",NA,"20241118T132242Z","RELEASE-2025" +"af413b63-21bd-4253-b015-bc7bab72b163","DELA_051.basePlot.dhp","D08","DELA","DELA_051","E2",2023-06-06 11:28:00,2023-06-06 13:39:00,"DHP.2023.23.DELA051","DHP.2023.23.DELA051.E2.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7625.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_051/understory/D08_7625.NEF",NA,"20241118T132242Z","RELEASE-2025" +"9401523d-909e-484e-a745-d034c746fade","DELA_051.basePlot.dhp","D08","DELA","DELA_051","W2",2023-06-06 11:28:00,2023-06-06 13:39:00,"DHP.2023.23.DELA051","DHP.2023.23.DELA051.W2.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7631.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_051/understory/D08_7631.NEF",NA,"20241118T132242Z","RELEASE-2025" +"d6b71ff1-f5c1-4dac-8abd-0c09eba08326","DELA_051.basePlot.dhp","D08","DELA","DELA_051","E10",2023-06-06 11:28:00,2023-06-06 13:39:00,"DHP.2023.23.DELA051","DHP.2023.23.DELA051.E10.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7627.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_051/understory/D08_7627.NEF",NA,"20241118T132242Z","RELEASE-2025" +"a836f36e-5c5f-4ba4-a1a1-577b30931238","DELA_051.basePlot.dhp","D08","DELA","DELA_051","S6",2023-06-06 11:28:00,2023-06-06 13:39:00,"DHP.2023.23.DELA051","DHP.2023.23.DELA051.S6.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7629.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_051/understory/D08_7629.NEF",NA,"20241118T132242Z","RELEASE-2025" +"82aa6855-e51e-4d99-81b5-89cbc7b26650","DELA_051.basePlot.dhp","D08","DELA","DELA_051","N10",2023-06-06 11:28:00,2023-06-06 13:39:00,"DHP.2023.23.DELA051","DHP.2023.23.DELA051.N10.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7623.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_051/understory/D08_7623.NEF",NA,"20241118T132242Z","RELEASE-2025" +"4d4e485f-19ea-4790-af50-9a10126c36f3","DELA_051.basePlot.dhp","D08","DELA","DELA_051","W6",2023-06-06 11:28:00,2023-06-06 13:39:00,"DHP.2023.23.DELA051","DHP.2023.23.DELA051.W6.UNDERSTORY","downward","shoulder","OK - no known exceptions","understory","D08_7632.NEF","https://storage.neonscience.org/neon-dhp-images/D08/2023/DELA23/DELA_051/understory/D08_7632.NEF",NA,"20241118T132242Z","RELEASE-2025" diff --git a/testdata/NEON_uri_testdata/10017_DELA_202306_RELEASE2025/variables_10017.csv b/testdata/NEON_uri_testdata/10017_DELA_202306_RELEASE2025/variables_10017.csv new file mode 100644 index 0000000..4dcaaea --- /dev/null +++ b/testdata/NEON_uri_testdata/10017_DELA_202306_RELEASE2025/variables_10017.csv @@ -0,0 +1,54 @@ +"table","fieldName","description","dataType","units","downloadPkg","pubFormat","primaryKey","categoricalCodeName" +"dhp_perimagefile","uid","Unique ID within NEON database; an identifier for the record","string",NA,"basic","asIs","N","" +"dhp_perimagefile","namedLocation","Name of the measurement location in the NEON database","string",NA,"basic","asIs","N","" +"dhp_perimagefile","domainID","Unique identifier of the NEON domain","string",NA,"basic","asIs","N","" +"dhp_perimagefile","siteID","NEON site code","string",NA,"basic","asIs","N","" +"dhp_perimagefile","plotID","Plot identifier (NEON site code_XXX)","string",NA,"basic","asIs","N","" +"dhp_perimagefile","pointID","Identifier for a point location","string",NA,"basic","LOV","N","dhp.pointID" +"dhp_perimagefile","startDate","The start date-time or interval during which an event occurred","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(round)","N","" +"dhp_perimagefile","endDate","The end date-time or interval during which an event occurred","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(round)","N","" +"dhp_perimagefile","sampleID","Identifier for sample","string",NA,"basic","asIs","N","" +"dhp_perimagefile","subsampleID","Unique identifier associated with each subsample per sampleID","string",NA,"basic","asIs","Y","" +"dhp_perimagefile","cameraOrientation","Direction camera was facing when image was acquired","string",NA,"basic","LOV","N","dhp.cameraOrientation" +"dhp_perimagefile","cameraPosition","Height camera was held when image was acquired","string",NA,"basic","LOV","N","dhp.cameraPosition" +"dhp_perimagefile","biophysicalCriteria","An indicator of whether sampling coincided with the intended biophysical conditions","string",NA,"basic","LOV","N","biophysicalCriteria" +"dhp_perimagefile","imageType","Subject matter contained within image","string",NA,"basic","LOV","N","dhp.imageType" +"dhp_perimagefile","imageFileName","Identifier for the image file assigned automatically by the camera","string",NA,"basic","asIs","N","" +"dhp_perimagefile","imageFileUrl","The URL identifying the external image file location","uri",NA,"basic","asIs","N","" +"dhp_perimagefile","downloadFileName","The name of the user-downloaded file that is linked to the record","string",NA,"expanded","asIs","N","" +"dhp_perimagefile","remarks","Technician notes; free text comments accompanying the record","string",NA,"basic","asIs","N","" +"dhp_perimagefile","dataQF","Data quality flag","string",NA,"expanded","asIs","N","" +"dhp_perimagefile","publicationDate","Date of data publication on the NEON data portal","dateTime",NA,"appended by stackByTable",NA,"N","" +"dhp_perimagefile","release","Identifier for data release","string",NA,"appended by stackByTable",NA,"N","" +"dhp_perplot","uid","Unique ID within NEON database; an identifier for the record","string",NA,"basic","asIs","N","" +"dhp_perplot","namedLocation","Name of the measurement location in the NEON database","string",NA,"basic","asIs","N","" +"dhp_perplot","domainID","Unique identifier of the NEON domain","string",NA,"basic","asIs","N","" +"dhp_perplot","siteID","NEON site code","string",NA,"basic","asIs","N","" +"dhp_perplot","plotID","Plot identifier (NEON site code_XXX)","string",NA,"basic","asIs","Y","" +"dhp_perplot","geodeticDatum","Model used to measure horizontal position on the earth","string",NA,"basic","UPPER","N","" +"dhp_perplot","decimalLatitude","The geographic latitude (in decimal degrees, WGS84) of the geographic center of the reference area","real","decimalDegree","basic","*.######(round)","N","" +"dhp_perplot","decimalLongitude","The geographic longitude (in decimal degrees, WGS84) of the geographic center of the reference area","real","decimalDegree","basic","*.######(round)","N","" +"dhp_perplot","coordinateUncertainty","The horizontal distance (in meters) from the given decimalLatitude and decimalLongitude describing the smallest circle containing the whole of the Location. Zero is not a valid value for this term","real","meter","basic","*.#(round)","N","" +"dhp_perplot","utmZone","UTM zone","string",NA,"basic","asIs","N","" +"dhp_perplot","elevation","Elevation (in meters) above sea level","real","meter","basic","*.#(round)","N","" +"dhp_perplot","elevationUncertainty","Uncertainty in elevation values (in meters)","real","meter","basic","*.#(round)","N","" +"dhp_perplot","slopeAspect","Representative azimuth of slope gradient (0-360 degrees)","real","degree","basic","*.#(round)","N","" +"dhp_perplot","slopeGradient","Representative inclination of slope in degrees","real","degree","basic","*.#(round)","N","" +"dhp_perplot","nlcdClass","National Land Cover Database Vegetation Type Name","string",NA,"basic","asIs","N","" +"dhp_perplot","plotType","NEON plot type in which sampling occurred: tower, distributed or gradient","string",NA,"basic","asIs","N","" +"dhp_perplot","plotSize","Size (in square meters) of the plot or grid","real","squareMeter","basic","integer","N","" +"dhp_perplot","startDate","The start date-time or interval during which an event occurred","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(round)","N","" +"dhp_perplot","endDate","The end date-time or interval during which an event occurred","dateTime",NA,"basic","yyyy-MM-dd'T'HH:mm'Z'(round)","N","" +"dhp_perplot","aopCollectDate","Date that remote sensing data from the NEON AOP were collected for the sample","dateTime",NA,"basic","yyyy-MM-dd(floor)","N","" +"dhp_perplot","samplingImpractical","Samples and/or measurements were not collected due to the indicated circumstance","string",NA,"basic","LOV","Y","samplingImpractical" +"dhp_perplot","biophysicalCriteria","An indicator of whether sampling coincided with the intended biophysical conditions","string",NA,"basic","LOV","N","biophysicalCriteria" +"dhp_perplot","eventID","An identifier for the set of information associated with the event, which includes information about the place and time of the event","string",NA,"basic","asIs","N","" +"dhp_perplot","sampleID","Identifier for sample","string",NA,"basic","asIs","Y","" +"dhp_perplot","samplingProtocolVersion","The NEON document number and version where detailed information regarding the sampling method used is available; format NEON.DOC.######vX","string",NA,"basic","LOV","N","dhp.samplingProtocolVersion" +"dhp_perplot","understoryHeight","Average height of the understory vegetation in a plot, assessed by visual survey to guide LAI photo collection","real","meter","basic","*.#(round)","N","" +"dhp_perplot","overstoryHeight","Average height of the overstory vegetation in a plot, assessed by visual survey to guide LAI photo collection","real","meter","basic","*.#(round)","N","" +"dhp_perplot","snowPresent","Categorical indicator of whether snow is present within the plot or sampling area","string",NA,"basic","LOV","N","Yes or No choice" +"dhp_perplot","remarks","Technician notes; free text comments accompanying the record","string",NA,"basic","asIs","N","" +"dhp_perplot","measuredBy","An identifier for the technician who measured or collected the data","string",NA,"basic","asIs","N","" +"dhp_perplot","recordedBy","An identifier for the technician who recorded the data","string",NA,"basic","asIs","N","" +"dhp_perplot","dataQF","Data quality flag","string",NA,"expanded","asIs","N","" diff --git a/tests/test_aop_download.py b/tests/test_aop_download.py index 323e456..7e56c0b 100644 --- a/tests/test_aop_download.py +++ b/tests/test_aop_download.py @@ -60,7 +60,7 @@ def test_invalid_dpid_format(self): invalid_dpid = "DP1.30001" with self.assertRaises( ValueError, - msg=f"{invalid_dpid} is not a properly formatted data product ID. The correct format is DP#.#####.00#", + msg=f"{invalid_dpid} is not a properly formatted NEON data product ID. The correct format is DP#.#####.00#", ): by_file_aop(dpid=invalid_dpid, site=self.site, year=self.year) @@ -71,7 +71,7 @@ def test_invalid_aop_dpid_pattern(self): invalid_aop_dpid = "DP1.20001.001" with self.assertRaises( ValueError, - msg=f"{invalid_aop_dpid} is not a valid AOP data product ID. AOP products follow the format DP#.300##.00#", + msg=f"{invalid_aop_dpid} is not a valid NEON AOP data product ID. AOP products follow the format DP#.300##.00#", ): by_file_aop(dpid=invalid_aop_dpid, site=self.site, year=self.year) @@ -83,7 +83,7 @@ def test_invalid_aop_dpid_suspended(self): # ' Valid AOP DPIDs are '): with self.assertRaises( ValueError, - msg=f"{suspended_aop_dpid} has been suspended and is not currently available, see https://www.neonscience.org/data-products/{suspended_aop_dpid} for more details.", + msg=f"NEON {suspended_aop_dpid} has been suspended and is not currently available, see https://www.neonscience.org/data-products/{suspended_aop_dpid} for more details.", ): by_file_aop(dpid=suspended_aop_dpid, site=self.site, year=self.year) @@ -94,7 +94,7 @@ def test_check_field_spectra_dpid(self): field_spectra_dpid = "DP1.30012.001" with self.assertRaises( ValueError, - msg=f"{field_spectra_dpid} is the Field spectral data product, which is published as tabular data. Use zipsByProduct() or loadByProduct() to download these data.", + msg=f"NEON {field_spectra_dpid} is the Field spectral data product, which is published as tabular data. Use zipsByProduct() or loadByProduct() to download these data.", ): by_file_aop(dpid=field_spectra_dpid, site=self.site, year=self.year) @@ -105,7 +105,7 @@ def test_invalid_site_format(self): invalid_site = "McRae" with self.assertRaises( ValueError, - msg=f"{invalid_site} is an invalid site format. A four-letter NEON site code is required. NEON sites codes can be found here: https://www.neonscience.org/field-sites/explore-field-sites", + msg=f"{invalid_site} is an invalid NEON site format. A four-letter NEON site code is required. NEON sites codes can be found here: https://www.neonscience.org/field-sites/explore-field-sites", ): by_file_aop(dpid=self.dpid, site=invalid_site, year=self.year) @@ -132,7 +132,7 @@ def test_invalid_year_format(self, year): """ with pytest.raises( ValueError, - match=f'{year} is an invalid year. Year is required in the format "2017" or 2017, eg. AOP data are available from 2013 to present.', + match=f'{year} is an invalid year. Year is required in the format "2017" or 2017, eg. NEON AOP data are available from 2013 to present.', ): by_file_aop(dpid=self.dpid, site=self.site, year=year) @@ -156,7 +156,7 @@ def test_collocated_terrestrial_site_message( with self.assertLogs(level="INFO") as cm: by_file_aop(dpid="DP3.30015.001", site=site, year=year, token=token) self.assertIn( - f"INFO:root:{site} is part of the flight box for {flightSite}. Downloading data from {flightSite}.", + f"INFO:root:{site} is part of the NEON flight box for {flightSite}. Downloading data from {flightSite}.", cm.output, ) @@ -177,7 +177,7 @@ def test_collocated_aquatic_site_message(self, year, site, flightSite, input_moc with self.assertLogs(level="INFO") as cm: by_file_aop(dpid=self.dpid, site=site, year=year, token=token) self.assertIn( - f"INFO:root:{site} is an aquatic site and is sometimes included in the flight box for {flightSite}. Aquatic sites are not always included in the flight coverage every year.\nDownloading data from {flightSite}. Check data to confirm coverage of {site}.", + f"INFO:root:{site} is a NEON aquatic site and is sometimes included in the flight box for {flightSite}. Aquatic sites are not always included in the flight coverage every year.\nDownloading data from {flightSite}. Check data to confirm coverage of {site}.", cm.output, ) @@ -188,7 +188,7 @@ def test_no_data_available_message(self): with self.assertLogs(level="INFO") as cm: by_file_aop(dpid="DP3.30015.001", site=self.site, year=2020) self.assertIn( - f"INFO:root:There are no {self.dpid} data available at the site {self.site} in 2020.\nTo display available dates for a given data product and site, use the function list_available_dates().", + f"INFO:root:There are no NEON {self.dpid} data available at the site {self.site} in 2020.\nTo display available dates for a given data product and site, use the function list_available_dates().", cm.output, ) @@ -200,7 +200,7 @@ def test_check_download_size_message(self, input_mock): result = by_file_aop(dpid=self.dpid, site=self.site, year=self.year) # Check that the function asked for confirmation to download and prints expected message. input_mock.assert_called_once_with( - "Continuing will download 128 files totaling approximately 93.1 MB. Do you want to proceed? (y/n) " + "Continuing will download 128 NEON data files totaling approximately 93.1 MB. Do you want to proceed? (y/n) " ) # Check that the function halted the download self.assertEqual(result, None) @@ -216,7 +216,7 @@ def test_all_provisional_no_data_available_message(self): with self.assertLogs(level="INFO") as cm: by_file_aop(dpid="DP3.30015.001", site="WLOU", year=2024) self.assertIn( - "INFO:root:No data files found. Available data may all be provisional. To download provisional data, use input parameter include_provisional=True.", + "INFO:root:No NEON data files found. Available data may all be provisional. To download provisional data, use input parameter include_provisional=True.", cm.output, ) @@ -232,7 +232,7 @@ def test_provisional_included_and_data_available_message(self, input_mock): dpid="DP3.30015.001", site="WLOU", year=2024, include_provisional=True ) self.assertIn( - "INFO:root:Provisional data are included. To exclude provisional data, use input parameter include_provisional=False.", + "INFO:root:Provisional NEON data are included. To exclude provisional data, use input parameter include_provisional=False.", cm.output, ) @@ -255,7 +255,7 @@ def test_invalid_dpid_format(self): invalid_dpid = "DP1.30001" with self.assertRaises( ValueError, - msg=f"{invalid_dpid} is not a properly formatted data product ID. The correct format is DP#.#####.00#", + msg=f"{invalid_dpid} is not a properly formatted NEON data product ID. The correct format is DP#.#####.00#", ): by_tile_aop( dpid=invalid_dpid, @@ -272,7 +272,7 @@ def test_invalid_aop_l3_dpid(self): invalid_aop_dpid = "DP1.30001.001" with self.assertRaises( ValueError, - msg=f"{invalid_aop_dpid} is not a valid Level 3 AOP data product ID. Level 3 AOP products follow the format DP3.300##.00#", + msg=f"NEON {invalid_aop_dpid} is not a valid Level 3 AOP data product ID. Level 3 AOP products follow the format DP3.300##.00#", ): by_tile_aop( dpid=invalid_aop_dpid, @@ -286,7 +286,7 @@ def test_check_field_spectra_dpid(self): field_spectra_dpid = "DP1.30012.001" with self.assertRaises( ValueError, - msg=f"{field_spectra_dpid} is the Field spectral data product, which is published as tabular data. Use zipsByProduct() or loadByProduct() to download these data.", + msg=f"NEON {field_spectra_dpid} is the Field spectral data product, which is published as tabular data. Use zipsByProduct() or loadByProduct() to download these data.", ): by_tile_aop( dpid=field_spectra_dpid, @@ -300,7 +300,7 @@ def test_invalid_site_format(self): invalid_site = "McRae" with self.assertRaises( ValueError, - msg=f"{invalid_site} is an invalid site format. A four-letter NEON site code is required. NEON sites codes can be found here: https://www.neonscience.org/field-sites/explore-field-sites", + msg=f"{invalid_site} is an invalid NEON site format. A four-letter NEON site code is required. NEON sites codes can be found here: https://www.neonscience.org/field-sites/explore-field-sites", ): by_tile_aop( dpid=self.dpid, @@ -336,7 +336,7 @@ def test_invalid_year_format(self, year): """ with pytest.raises( ValueError, - match=f'{year} is an invalid year. Year is required in the format "2017" or 2017, eg. AOP data are available from 2013 to present.', + match=f'{year} is an invalid year. Year is required in the format "2017" or 2017, eg. NEON AOP data are available from 2013 to present.', ): by_tile_aop( dpid=self.dpid, @@ -366,7 +366,7 @@ def test_collocated_terrestrial_site_message( with self.assertLogs(level="INFO") as cm: by_tile_aop(dpid=self.dpid, site=site, year=year, easting=[], northing=[]) self.assertIn( - f"INFO:root:{site} is part of the flight box for {flightSite}. Downloading data from {flightSite}.", + f"INFO:root:{site} is part of the NEON flight box for {flightSite}. Downloading data from {flightSite}.", cm.output, ) @@ -387,7 +387,7 @@ def test_collocated_aquatic_site_message(self, year, site, flightSite, input_moc with self.assertLogs(level="INFO") as cm: by_tile_aop(dpid=self.dpid, site=site, year=year, easting=[], northing=[]) self.assertIn( - f"INFO:root:{site} is an aquatic site and is sometimes included in the flight box for {flightSite}. Aquatic sites are not always included in the flight coverage every year.\nDownloading data from {flightSite}. Check data to confirm coverage of {site}.", + f"INFO:root:{site} is a NEON aquatic site and is sometimes included in the flight box for {flightSite}. Aquatic sites are not always included in the flight coverage every year.\nDownloading data from {flightSite}. Check data to confirm coverage of {site}.", cm.output, ) @@ -404,7 +404,7 @@ def test_no_data_available_message(self): northing=self.northing, ) self.assertIn( - f"INFO:root:There are no DP3.30015.001 data available at the site {self.site} in 2020.\nTo display available dates for a given data product and site, use the function list_available_dates().", + f"INFO:root:There are no NEON DP3.30015.001 data available at the site {self.site} in 2020.\nTo display available dates for a given data product and site, use the function list_available_dates().", cm.output, ) # 'INFO:root:There are no data available at the selected site and year.', cm.output) @@ -422,7 +422,7 @@ def test_no_data_files_found_message(self): northing=4900000, ) self.assertIn( - f"INFO:root:There are no DP3.30015.001 data available at the site {self.site} in 2020.\nTo display available dates for a given data product and site, use the function list_available_dates().", + f"INFO:root:There are no NEON DP3.30015.001 data available at the site {self.site} in 2020.\nTo display available dates for a given data product and site, use the function list_available_dates().", cm.output, ) @@ -440,7 +440,7 @@ def test_check_download_size_message(self, input_mock): ) # Check that the function asked for confirmation to download and prints expected message. input_mock.assert_called_once_with( - "Continuing will download 7 files totaling approximately 3.9 MB. Do you want to proceed? (y/n) " + "Continuing will download 7 NEON data files totaling approximately 3.9 MB. Do you want to proceed? (y/n) " ) # Check that the function halted the download self.assertEqual(result, None) @@ -463,7 +463,7 @@ def test_all_provisional_no_data_available_message(self): northing=self.northing, ) self.assertIn( - "INFO:root:No data files found. Available data may all be provisional. To download provisional data, use input parameter include_provisional=True.", + "INFO:root:No NEON data files found. Available data may all be provisional. To download provisional data, use input parameter include_provisional=True.", cm.output, ) @@ -484,7 +484,7 @@ def test_provisional_included_and_data_available_message(self, input_mock): northing=self.northing, ) self.assertIn( - "INFO:root:Provisional data are included. To exclude provisional data, use input parameter include_provisional=False.", + "INFO:root:Provisional NEON data are included. To exclude provisional data, use input parameter include_provisional=False.", cm.output, ) diff --git a/tests/test_files_by_uri.py b/tests/test_files_by_uri.py new file mode 100644 index 0000000..cde2cfc --- /dev/null +++ b/tests/test_files_by_uri.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +""" +Created on 14 Apr 2025 + +@author: Zachary Nickerson + +Unit tests for files_by_uri() + +""" + +# import required packages +from src.neonutilities.files_by_uri import files_by_uri +import os +import shutil + +def test_files_by_uri_NEF(): + """ + Test that the function works for NEF files available from DP1.10017.001 (tabular data saved in testdata) + """ + testdir = "./testdata/NEON_uri_testdata/10017_DELA_202306_RELEASE2025" + files_by_uri(testdir, + savepath=testdir, + check_size=False, + unzip=True, + save_zipped_files=False, + progress=True) + # There should be 36 .NEF files in the directory + files = os.listdir(os.path.join(testdir,"GCS_files")) + nef_files = [file for file in files if file.lower().endswith('.nef')] + assert len(nef_files) == 36 + # Remove the .NEF files saved in the testdir + shutil.rmtree(os.path.join(testdir,"GCS_files")) + +def test_files_by_uri_ZIP(): + """ + Test that the function works for ZIP files available from DP4.00131.001 (tabular data saved in testdata) + """ + testdir = "./testdata/NEON_uri_testdata/00131_allSites_2022_RELEASE2025" + files_by_uri(testdir, + savepath=testdir, + check_size=False, + unzip=False, + save_zipped_files=False, + progress=True) + # There should be 21 .ZIP files in the directory + files = os.listdir(os.path.join(testdir,"GCS_files")) + zip_files = [file for file in files if file.lower().endswith('.zip')] + assert len(zip_files) == 21 + # Remove the .NEF files saved in the testdir + shutil.rmtree(os.path.join(testdir,"GCS_files")) + +def test_files_by_uri_ZIP_unzip(): + """ + Test that the function works for ZIP files available from DP4.00131.001, and unzip the files (tabular data saved in testdata) + """ + testdir = "./testdata/NEON_uri_testdata/00131_allSites_2022_RELEASE2025" + files_by_uri(testdir, + savepath=testdir, + check_size=False, + unzip=True, + save_zipped_files=False, + progress=True) + # There should be 294 files that are not .CSV in the directory + files = os.listdir(os.path.join(testdir,"GCS_files")) + geo_files = [file for file in files if not file.lower().endswith('.csv')] + assert len(geo_files) == 294 + # Remove the .NEF files saved in the testdir + shutil.rmtree(os.path.join(testdir,"GCS_files")) + +# Potentially add a test for the microbial file types \ No newline at end of file diff --git a/tests/test_zips_by_product.py b/tests/test_zips_by_product.py index 3618a5d..0550932 100644 --- a/tests/test_zips_by_product.py +++ b/tests/test_zips_by_product.py @@ -26,7 +26,7 @@ def test_zips_by_product_dpid(): ) assert ( str(exc_info.value) - == "DP1.444.001 is not a properly formatted data product ID. The correct format is DP#.#####.00#" + == "DP1.444.001 is not a properly formatted NEON data product ID. The correct format is DP#.#####.00#" ) @@ -44,7 +44,7 @@ def test_zips_by_product_site(caplog): ) assert any( - "There are no data at the selected sites." in record.message + "There are no NEON DP1.10003.001 data at the selected sites." in record.message for record in caplog.records )