diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..14d7c2d --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,3 @@ +[isort] +multi_line_output = 3 +include_trailing_comma = true diff --git a/README.md b/README.md index 28bf837..77092db 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,98 @@ -# stactools-package - -Template repostitory for [stactools](https://github.com/stac-utils/stactools) packages. - -## How to use - -1. Clone this template repository as your package name, e.g. `landsat`. - This name should be short, memorable, and a valid Python package name (i.e. it shouldn't start with a number, etc). - It can, however, include a hyphen, in which case the name for Python imports will be the underscored version, e.g. `landsat-8` goes to `stactools.landsat_8`. - Your name will be used on PyPI to publish the package in the stactools namespace, e.g. `stactools-landsat`. -2. Change into the top-level directory of your package and run `scripts/rename`. - This will update _most_ of the files in the repository with your new package name. - You'll have to manually update `setup.cfg` and `README.md`. -3. Update `setup.cfg` with your package name, description, and such. -4. Rewrite this README to provide information about how to use your package. -5. Update the LICENSE with your company's information (or whomever holds the copyright). -6. Run `sphinx-quickstart` in the `docs` directory to create the documentation template. -7. Update `docs/installation_and_basic_usage.ipynb` to provide an interactive notebook to help users get started. Include the following badge at the top of the README to launch the notebook: [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/stactools-packages/template/main?filepath=docs/installation_and_basic_usage.ipynb). Be sure to modify the badge href to match your package repo. +# stactools-package gpw + +- Name: Gridded Population of the World (GPW) v4, Population Count, v4.11 +- Package: `stactools.gpw` +- PyPI: https://pypi.org/project/stactools-gpw/ +- Owner: @sparkgeo +- Dataset homepage: https://sedac.ciesin.columbia.edu/data/collection/gpw-v4/sets/browse +- STAC extensions used: + - [file](https://github.com/stac-extensions/file/) + - [item-assets](https://github.com/stac-extensions/item-assets/) + - [proj](https://github.com/stac-extensions/projection/) + - [raster](https://github.com/stac-extensions/raster/) + - [scientific](https://github.com/stac-extensions/scientific/) + +The Gridded Population of the World, Version 4 (GPWv4) Revision 11 Population Datasets (population count, UN WPP-adjusted population count, population density, and UN WPP-adjusted population density) consist of estimates of human population (number of persons per pixel) and population density (number of persons per square kilometer), consistent with national censuses and population registers, for the years 2000, 2005, 2010, 2015, and 2020. Where indicated by name, values have been adjusted to match the 2015 Revision of the United Nation's World Population Prospects (UN WPP) country totals. + +The Gridded Population of the World, Version 4 (GPWv4) Revision 11 Ancillary Datasets (data quality indicators, basic demographic characteristics, land and water area, and national identifier grid) provide context for the population count and density rasters found within the Gridded Population of the World Population Collection. + +## Usage + +### Using the CLI + +```bash +mkdir example_output +# Create a COG - creates example_output/gpw_v4_population_count_rev11_2000_30_sec_cog.tif +stac gpw create-cog -d example_output -s tests/data-files/raw/population/gpw_v4_population_count_rev11_2000_30_sec.tif +# Create a COG - first expanding the dataset bbox, then retiling to 10000 x 10000 pixel COG files. Creates several tif files within example_output. +stac gpw create-cog -d example_output -s tests/data-files/raw/population/gpw_v4_population_count_rev11_2000_30_sec.tif -t -e -180 -90 180 90 +# Create a population STAC Item - creates example_output/gpw_v4_rev11_2000_30_sec_2_1.json +stac gpw create-pop-item \ + -d example_output \ + -c \ + tests/data-files/tiles/population/gpw_v4_population_count_rev11_2000_30_sec_2_1_cog.tif \ + tests/data-files/tiles/population/gpw_v4_population_count_adjusted_to_2015_unwpp_country_totals_rev11_2000_30_sec_2_1_cog.tif \ + tests/data-files/tiles/population/gpw_v4_population_density_rev11_2000_30_sec_2_1_cog.tif \ + tests/data-files/tiles/population/gpw_v4_population_density_adjusted_to_2015_unwpp_country_totals_rev11_2000_30_sec_2_1_cog.tif +# Create an ancillary STAC Item - creates example_output/gpw_v4_rev11_30_sec_2_1.json +stac gpw create-anc-item \ + -d example_output \ + -c \ + tests/data-files/tiles/ancillary/gpw_v4_data_quality_indicators_rev11_context_30_sec_2_1_cog.tif \ + tests/data-files/tiles/ancillary/gpw_v4_data_quality_indicators_rev11_mean_adminunitarea_30_sec_2_1_cog.tif \ + tests/data-files/tiles/ancillary/gpw_v4_data_quality_indicators_rev11_watermask_30_sec_2_1_cog.tif \ + tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopbt_2010_cntm_30_sec_2_1_cog.tif \ + tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopbt_2010_dens_30_sec_2_1_cog.tif \ + tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopft_2010_cntm_30_sec_2_1_cog.tif \ + tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopft_2010_dens_30_sec_2_1_cog.tif \ + tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopmt_2010_cntm_30_sec_2_1_cog.tif \ + tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopmt_2010_dens_30_sec_2_1_cog.tif \ + tests/data-files/tiles/ancillary/gpw_v4_land_water_area_rev11_landareakm_30_sec_2_1_cog.tif \ + tests/data-files/tiles/ancillary/gpw_v4_land_water_area_rev11_waterareakm_30_sec_expanded_2_1_cog.tif \ + tests/data-files/tiles/ancillary/gpw_v4_national_identifier_grid_rev11_30_sec_2_1_cog.tif + +# Create a population STAC Collection - creates example_output/collection.json +stac gpw create-pop-collection -d example_output/collection.json + +# Create a population STAC Collection - creates example_output/collection.json +stac gpw create-anc-collection -d example_output/collection.json +``` + +### As a python module + +```python +from stactools.gpw import cog, stac + +# Create a population (or ancillary) STAC Collection +stac.create_pop_collection("example_output/collection.json") +# stac.create_anc_collection("example_output/collection.json") + +# Create a COG (optional parameters to retile or expand tiled output) +cog.create_cog("/path/to/local.tif", "/path/to/output_directory") + +# Create a population STAC Item +cog_list = [ + Path to population count COG + Path to adjusted population count COG + Path to population density COG + Path to adjusted population density COG +] +stac.create_pop_item(*cog_list) + +# Create an ancillary STAC Item +cog_list = [ + Path to Data Quality Indicators - Data Context COG + Path to Data Quality Indicators - Mean Administrative Unit Area COG + Path to Data Quality Indicators - Water Mask COG + Path to Basic Demographic Characteristics - Ttl Pop Count + Path to Basic Demographic Characteristics - Ttl Pop Density + Path to Basic Demographic Characteristics - Female Count + Path to Basic Demographic Characteristics - Female Density + Path to Basic Demographic Characteristics - Male Count + Path to Basic Demographic Characteristics - Male Density + Path to Land Area COG + Path to Water Area COG + Path to National Identifier Grid COG +] +stac.create_anc_item(*cog_list) +``` diff --git a/examples/ancillary/collection.json b/examples/ancillary/collection.json new file mode 100644 index 0000000..3579488 --- /dev/null +++ b/examples/ancillary/collection.json @@ -0,0 +1,142 @@ +{ + "type": "Collection", + "id": "GPW-ancillary", + "stac_version": "1.0.0", + "description": "The Gridded Population of the World, Version 4 (GPWv4) Revision 11 Ancillary Datasets (data quality indicators, basic demographic characteristics, land and water area, and national identifier grid) provide context for the population count and density rasters found within the Gridded Population of the World Population Collection.", + "links": [ + { + "rel": "root", + "href": "./collection.json", + "type": "application/json" + }, + { + "rel": "license", + "href": "https://creativecommons.org/licenses/by/4.0/legalcode", + "title": "Creative Commons Attribution 4.0 International" + }, + { + "rel": "self", + "href": "path/to/stactools-packages/gpw/examples/ancillary/collection.json", + "type": "application/json" + } + ], + "stac_extensions": [ + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json", + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json" + ], + "item_assets": { + "anc_dqi_context": { + "title": "Data Quality Indicators - Data Context", + "description": "Gridded Population of the World, Version 4 (GPWv4): Data Quality Indicators - Data Context, 30 arc-seconds, 1km resolution", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "sci:doi": "10.7927/H42Z13KG", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Data Quality Indicators, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H42Z13KG. Accessed 22 Octover 2021." + }, + "anc_dqi_admin": { + "title": "Data Quality Indicators - Mean Administrative Unit Area", + "description": "Gridded Population of the World, Version 4 (GPWv4): Data Quality Indicators - Mean Administrative Unit Area, 30 arc-seconds, 1km resolution", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "sci:doi": "10.7927/H42Z13KG", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Data Quality Indicators, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H42Z13KG. Accessed 22 October 2021." + }, + "anc_dqi_watermask": { + "title": "Data Quality Indicators - Water Mask", + "description": "Gridded Population of the World, Version 4 (GPWv4): Data Quality Indicators - Water Mask, 30 arc-seconds, 1km resolution", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "sci:doi": "10.7927/H42Z13KG", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Data Quality Indicators, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H42Z13KG. Accessed 22 October 2021." + }, + "anc_bdc_bt_count": { + "title": "Basic Demographic Characteristics - Total Population Count", + "description": "Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics - Total Population Count, 30 arc-seconds, 1km resolution", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "sci:doi": "10.7927/H46M34XX", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H46M34XX. Accessed 22 October 2021." + }, + "anc_bdc_bt_density": { + "title": "Basic Demographic Characteristics - Total Population Density", + "description": "Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics - Total Population Density, 30 arc-seconds, 1km resolution", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "sci:doi": "10.7927/H46M34XX", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H46M34XX. Accessed 22 October 2021." + }, + "anc_bdc_ft_count": { + "title": "Basic Demographic Characteristics - Female Count", + "description": "Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics - Female Count, 30 arc-seconds, 1km resolution", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "sci:doi": "10.7927/H46M34XX", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H46M34XX. Accessed 22 October 2021." + }, + "anc_bdc_ft_density": { + "title": "Basic Demographic Characteristics - Female Density", + "description": "Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics - Female Density, 30 arc-seconds, 1km resolution", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "sci:doi": "10.7927/H46M34XX", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H46M34XX. Accessed 22 October 2021." + }, + "anc_bdc_mt_count": { + "title": "Basic Demographic Characteristics - Male Count", + "description": "Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics - Male Count, 30 arc-seconds, 1km resolution", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "sci:doi": "10.7927/H46M34XX", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H46M34XX. Accessed 22 October 2021." + }, + "anc_bdc_mt_density": { + "title": "Basic Demographic Characteristics - Male Density", + "description": "Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics - Male Density, 30 arc-seconds, 1km resolution", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "sci:doi": "10.7927/H46M34XX", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H46M34XX. Accessed 22 October 2021." + }, + "anc_land_area": { + "title": "Land Area", + "description": "Gridded Population of the World, Version 4 (GPWv4): Land Area, 30 arc-seconds, 1km resolution", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "sci:doi": "10.7927/H4Z60M4Z", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Land and Water Area, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H4Z60M4Z. Accessed 22 October 2021." + }, + "anc_water_area": { + "title": "Water Area", + "description": "Gridded Population of the World, Version 4 (GPWv4): Water Area, 30 arc-seconds, 1km resolution", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "sci:doi": "10.7927/H4Z60M4Z", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Land and Water Area, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H4Z60M4Z. Accessed 22 October 2021." + }, + "anc_nat_id_grid": { + "title": "National Identifier Grid", + "description": "Gridded Population of the World, Version 4 (GPWv4): UN WPP-Adjusted Population Density, 30 arc-seconds, 1km resolution", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "sci:doi": "10.7927/H4TD9VDP", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): National Identifier Grid, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H4TD9VDP. Accessed 22 October 2021." + } + }, + "title": "Gridded Population of the World (GPW) v4.11, Ancillary Datasets", + "extent": { + "spatial": { + "bbox": [[-180.0, 90.0, 180.0, -90.0]] + }, + "temporal": { + "interval": [["2010-01-01T00:00:00Z", "2011-01-01T00:00:00Z"]] + } + }, + "license": "CC-BY-4.0", + "providers": [ + { + "name": "Center for International Earth Science Information Network - CIESIN - Columbia University", + "roles": ["producer", "processor", "host"], + "url": "https://sedac.ciesin.columbia.edu/data/set/gpw-v4-population-count-rev11" + } + ] +} diff --git a/examples/ancillary/gpw_v4_rev11_30_sec_2_1.json b/examples/ancillary/gpw_v4_rev11_30_sec_2_1.json new file mode 100644 index 0000000..727b220 --- /dev/null +++ b/examples/ancillary/gpw_v4_rev11_30_sec_2_1.json @@ -0,0 +1,237 @@ +{ + "type": "Feature", + "stac_version": "1.0.0", + "id": "gpw_v4_rev11_30_sec_2_1", + "properties": { + "description": "The Gridded Population of the World, Version 4 (GPWv4) Revision 11 Ancillary Datasets (data quality indicators, basic demographic characteristics, land and water area, and national identifier grid) provide context for the population count and density rasters found within the Gridded Population of the World Population Collection.", + "proj:epsg": 4326, + "proj:bbox": [-180.0, 90.0, 180.0, -90.0], + "proj:transform": [ + 0.00833333333333333, 0.0, -180.0, 0.0, -0.00833333333333333, + 6.658333333333374, 0.0, 0.0, 1.0 + ], + "proj:shape": [10001, 10001], + "version": "4.11", + "datetime": "2010-07-01T00:00:00Z" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [180.0, 90.0], + [180.0, -90.0], + [-180.0, -90.0], + [-180.0, 90.0], + [180.0, 90.0] + ] + ] + }, + "links": [ + { + "rel": "self", + "href": "path/to/stactools-packages/gpw/examples/ancillary/gpw_v4_rev11_30_sec_2_1.json", + "type": "application/json" + } + ], + "assets": { + "anc_dqi_context": { + "href": "../../tests/data-files/tiles/ancillary/gpw_v4_data_quality_indicators_rev11_context_30_sec_2_1_cog.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Data Quality Indicators - Data Context", + "description": "Gridded Population of the World, Version 4 (GPWv4): Data Quality Indicators - Data Context, 30 arc-seconds, 1km resolution", + "sci:doi": "10.7927/H42Z13KG", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Data Quality Indicators, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H42Z13KG. Accessed 22 Octover 2021.", + "file:size": 829474, + "raster:bands": [ + { + "sampling": "area", + "data_type": "int16" + } + ], + "roles": ["data"] + }, + "anc_dqi_admin": { + "href": "../../tests/data-files/tiles/ancillary/gpw_v4_data_quality_indicators_rev11_mean_adminunitarea_30_sec_2_1_cog.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Data Quality Indicators - Mean Administrative Unit Area", + "description": "Gridded Population of the World, Version 4 (GPWv4): Data Quality Indicators - Mean Administrative Unit Area, 30 arc-seconds, 1km resolution", + "sci:doi": "10.7927/H42Z13KG", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Data Quality Indicators, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H42Z13KG. Accessed 22 October 2021.", + "file:size": 2947913, + "raster:bands": [ + { + "sampling": "area", + "data_type": "float32" + } + ], + "roles": ["data"] + }, + "anc_dqi_watermask": { + "href": "../../tests/data-files/tiles/ancillary/gpw_v4_data_quality_indicators_rev11_watermask_30_sec_2_1_cog.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Data Quality Indicators - Water Mask", + "description": "Gridded Population of the World, Version 4 (GPWv4): Data Quality Indicators - Water Mask, 30 arc-seconds, 1km resolution", + "sci:doi": "10.7927/H42Z13KG", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Data Quality Indicators, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H42Z13KG. Accessed 22 October 2021.", + "file:size": 819980, + "raster:bands": [ + { + "sampling": "area", + "data_type": "int16" + } + ], + "roles": ["data"] + }, + "anc_bdc_bt_count": { + "href": "../../tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopbt_2010_cntm_30_sec_2_1_cog.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Basic Demographic Characteristics - Total Population Count", + "description": "Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics - Total Population Count, 30 arc-seconds, 1km resolution", + "sci:doi": "10.7927/H46M34XX", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H46M34XX. Accessed 22 October 2021.", + "file:size": 3078792, + "raster:bands": [ + { + "sampling": "area", + "data_type": "float32" + } + ], + "roles": ["data"] + }, + "anc_bdc_bt_density": { + "href": "../../tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopbt_2010_dens_30_sec_2_1_cog.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Basic Demographic Characteristics - Total Population Density", + "description": "Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics - Total Population Density, 30 arc-seconds, 1km resolution", + "sci:doi": "10.7927/H46M34XX", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H46M34XX. Accessed 22 October 2021.", + "file:size": 2968195, + "raster:bands": [ + { + "sampling": "area", + "data_type": "float32" + } + ], + "roles": ["data"] + }, + "anc_bdc_ft_count": { + "href": "../../tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopft_2010_cntm_30_sec_2_1_cog.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Basic Demographic Characteristics - Female Count", + "description": "Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics - Female Count, 30 arc-seconds, 1km resolution", + "sci:doi": "10.7927/H46M34XX", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H46M34XX. Accessed 22 October 2021.", + "file:size": 3078444, + "raster:bands": [ + { + "sampling": "area", + "data_type": "float32" + } + ], + "roles": ["data"] + }, + "anc_bdc_ft_density": { + "href": "../../tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopft_2010_dens_30_sec_2_1_cog.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Basic Demographic Characteristics - Female Density", + "description": "Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics - Female Density, 30 arc-seconds, 1km resolution", + "sci:doi": "10.7927/H46M34XX", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H46M34XX. Accessed 22 October 2021.", + "file:size": 2968878, + "raster:bands": [ + { + "sampling": "area", + "data_type": "float32" + } + ], + "roles": ["data"] + }, + "anc_bdc_mt_count": { + "href": "../../tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopmt_2010_cntm_30_sec_2_1_cog.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Basic Demographic Characteristics - Male Count", + "description": "Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics - Male Count, 30 arc-seconds, 1km resolution", + "sci:doi": "10.7927/H46M34XX", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H46M34XX. Accessed 22 October 2021.", + "file:size": 3078301, + "raster:bands": [ + { + "sampling": "area", + "data_type": "float32" + } + ], + "roles": ["data"] + }, + "anc_bdc_mt_density": { + "href": "../../tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopmt_2010_dens_30_sec_2_1_cog.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Basic Demographic Characteristics - Male Density", + "description": "Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics - Male Density, 30 arc-seconds, 1km resolution", + "sci:doi": "10.7927/H46M34XX", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H46M34XX. Accessed 22 October 2021.", + "file:size": 2968465, + "raster:bands": [ + { + "sampling": "area", + "data_type": "float32" + } + ], + "roles": ["data"] + }, + "anc_land_area": { + "href": "../../tests/data-files/tiles/ancillary/gpw_v4_land_water_area_rev11_landareakm_30_sec_2_1_cog.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Land Area", + "description": "Gridded Population of the World, Version 4 (GPWv4): Land Area, 30 arc-seconds, 1km resolution", + "sci:doi": "10.7927/H4Z60M4Z", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Land and Water Area, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H4Z60M4Z. Accessed 22 October 2021.", + "file:size": 3085350, + "raster:bands": [ + { + "sampling": "area", + "data_type": "float32" + } + ], + "roles": ["data"] + }, + "anc_water_area": { + "href": "../../tests/data-files/tiles/ancillary/gpw_v4_land_water_area_rev11_waterareakm_30_sec_expanded_2_1_cog.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Water Area", + "description": "Gridded Population of the World, Version 4 (GPWv4): Water Area, 30 arc-seconds, 1km resolution", + "sci:doi": "10.7927/H4Z60M4Z", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Land and Water Area, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H4Z60M4Z. Accessed 22 October 2021.", + "file:size": 3070510, + "raster:bands": [ + { + "sampling": "area", + "data_type": "float32" + } + ], + "roles": ["data"] + }, + "anc_nat_id_grid": { + "href": "../../tests/data-files/tiles/ancillary/gpw_v4_national_identifier_grid_rev11_30_sec_2_1_cog.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "National Identifier Grid", + "description": "Gridded Population of the World, Version 4 (GPWv4): UN WPP-Adjusted Population Density, 30 arc-seconds, 1km resolution", + "sci:doi": "10.7927/H4TD9VDP", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): National Identifier Grid, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H4TD9VDP. Accessed 22 October 2021.", + "file:size": 833761, + "raster:bands": [ + { + "sampling": "area", + "data_type": "int16" + } + ], + "roles": ["data"] + } + }, + "bbox": [-180.0, 90.0, 180.0, -90.0], + "stac_extensions": [ + "https://stac-extensions.github.io/projection/v1.0.0/schema.json", + "https://stac-extensions.github.io/version/v1.0.0/schema.json", + "https://stac-extensions.github.io/file/v2.0.0/schema.json", + "https://stac-extensions.github.io/raster/v1.0.0/schema.json" + ] +} diff --git a/examples/population/collection.json b/examples/population/collection.json new file mode 100644 index 0000000..05795ef --- /dev/null +++ b/examples/population/collection.json @@ -0,0 +1,78 @@ +{ + "type": "Collection", + "id": "GPW-population", + "stac_version": "1.0.0", + "description": "The Gridded Population of the World, Version 4 (GPWv4) Revision 11 Population Datasets (population count, UN WPP-adjusted population count, population density, and UN WPP-adjusted population density) consist of estimates of human population (number of persons per pixel) and population density (number of persons per square kilometer), consistent with national censuses and population registers, for the years 2000, 2005, 2010, 2015, and 2020. Where indicated by name, values have been adjusted to match the 2015 Revision of the United Nation's World Population Prospects (UN WPP) country totals.", + "links": [ + { + "rel": "root", + "href": "./collection.json", + "type": "application/json" + }, + { + "rel": "license", + "href": "https://creativecommons.org/licenses/by/4.0/legalcode", + "title": "Creative Commons Attribution 4.0 International" + }, + { + "rel": "self", + "href": "path/to/stactools-packages/gpw/examples/population/collection.json", + "type": "application/json" + } + ], + "stac_extensions": [ + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json", + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json" + ], + "item_assets": { + "pop_count": { + "title": "Population Count", + "description": "Gridded Population of the World, Version 4 (GPWv4): Population Count, 30 arc-seconds, 1km resolution", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "sci:doi": "10.7927/H4JW8BX5", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Population Count, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H4JW8BX5. Accessed 22 October 2021." + }, + "pop_count_adj": { + "title": "UN WPP-Adjusted Population Count", + "description": "Gridded Population of the World, Version 4 (GPWv4): UN WPP-Adjusted Population Count, 30 arc-seconds, 1km resolution", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "sci:doi": "10.7927/H4PN93PB", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Population Count Adjusted to Match 2015 Revision of UN WPP Country Totals, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H4PN93PB. Accessed 22 October 2021." + }, + "pop_density": { + "title": "Population Density", + "description": "Gridded Population of the World, Version 4 (GPWv4): Population Density, 30 arc-seconds, 1km resolution", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "sci:doi": "10.7927/H49C6VHW", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Population Density, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H49C6VHW. Accessed 22 October 2021." + }, + "pop_density_adj": { + "title": "UN WPP-Adjusted Population Density", + "description": "Gridded Population of the World, Version 4 (GPWv4): UN WPP-Adjusted Population Density, 30 arc-seconds, 1km resolution", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "sci:doi": "10.7927/H4F47M65", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Population Density Adjusted to Match 2015 Revision UN WPP Country Totals, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H4F47M65. Accessed 22 October 2021." + } + }, + "title": "Gridded Population of the World (GPW) v4.11, Population Datasets", + "extent": { + "spatial": { + "bbox": [[-180.0, 90.0, 180.0, -90.0]] + }, + "temporal": { + "interval": [["2000-01-01T00:00:00Z", "2020-01-01T00:00:00Z"]] + } + }, + "license": "CC-BY-4.0", + "providers": [ + { + "name": "Center for International Earth Science Information Network - CIESIN - Columbia University", + "roles": ["producer", "processor", "host"], + "url": "https://sedac.ciesin.columbia.edu/data/set/gpw-v4-population-count-rev11" + } + ] +} diff --git a/examples/population/gpw_v4_rev11_2000_30_sec_2_1.json b/examples/population/gpw_v4_rev11_2000_30_sec_2_1.json new file mode 100644 index 0000000..4000306 --- /dev/null +++ b/examples/population/gpw_v4_rev11_2000_30_sec_2_1.json @@ -0,0 +1,111 @@ +{ + "type": "Feature", + "stac_version": "1.0.0", + "id": "gpw_v4_rev11_2000_30_sec_2_1", + "properties": { + "description": "The Gridded Population of the World, Version 4 (GPWv4) Revision 11 Population Datasets (population count, UN WPP-adjusted population count, population density, and UN WPP-adjusted population density) consist of estimates of human population (number of persons per pixel) and population density (number of persons per square kilometer), consistent with national censuses and population registers, for the years 2000, 2005, 2010, 2015, and 2020. Where indicated by name, values have been adjusted to match the 2015 Revision of the United Nation's World Population Prospects (UN WPP) country totals.", + "start_datetime": "2000-01-01T00:00:00Z", + "end_datetime": "2005-01-01T00:00:00Z", + "proj:epsg": 4326, + "proj:bbox": [-180.0, 90.0, 180.0, -90.0], + "proj:transform": [ + 0.00833333333333333, 0.0, -180.0, 0.0, -0.00833333333333333, + 6.658333333333374, 0.0, 0.0, 1.0 + ], + "proj:shape": [10001, 10001], + "version": "4.11", + "datetime": "2000-01-01T00:00:00Z" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [180.0, 90.0], + [180.0, -90.0], + [-180.0, -90.0], + [-180.0, 90.0], + [180.0, 90.0] + ] + ] + }, + "links": [ + { + "rel": "self", + "href": "path/to/stactools-packages/gpw/examples/population/gpw_v4_rev11_2000_30_sec_2_1.json", + "type": "application/json" + } + ], + "assets": { + "pop_count": { + "href": "../../tests/data-files/tiles/population/gpw_v4_population_count_rev11_2000_30_sec_2_1_cog.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Population Count", + "description": "Gridded Population of the World, Version 4 (GPWv4): Population Count, 30 arc-seconds, 1km resolution", + "sci:doi": "10.7927/H4JW8BX5", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Population Count, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H4JW8BX5. Accessed 22 October 2021.", + "file:size": 3082138, + "raster:bands": [ + { + "sampling": "area", + "data_type": "float32" + } + ], + "roles": ["data"] + }, + "pop_count_adj": { + "href": "../../tests/data-files/tiles/population/gpw_v4_population_count_adjusted_to_2015_unwpp_country_totals_rev11_2000_30_sec_2_1_cog.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "UN WPP-Adjusted Population Count", + "description": "Gridded Population of the World, Version 4 (GPWv4): UN WPP-Adjusted Population Count, 30 arc-seconds, 1km resolution", + "sci:doi": "10.7927/H4PN93PB", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Population Count Adjusted to Match 2015 Revision of UN WPP Country Totals, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H4PN93PB. Accessed 22 October 2021.", + "file:size": 3082134, + "raster:bands": [ + { + "sampling": "area", + "data_type": "float32" + } + ], + "roles": ["data"] + }, + "pop_density": { + "href": "../../tests/data-files/tiles/population/gpw_v4_population_density_adjusted_to_2015_unwpp_country_totals_rev11_2000_30_sec_2_1_cog.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "Population Density", + "description": "Gridded Population of the World, Version 4 (GPWv4): Population Density, 30 arc-seconds, 1km resolution", + "sci:doi": "10.7927/H49C6VHW", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Population Density, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H49C6VHW. Accessed 22 October 2021.", + "file:size": 3031272, + "raster:bands": [ + { + "sampling": "area", + "data_type": "float32" + } + ], + "roles": ["data"] + }, + "pop_density_adj": { + "href": "../../tests/data-files/tiles/population/gpw_v4_population_density_rev11_2000_30_sec_2_1_cog.tif", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "title": "UN WPP-Adjusted Population Density", + "description": "Gridded Population of the World, Version 4 (GPWv4): UN WPP-Adjusted Population Density, 30 arc-seconds, 1km resolution", + "sci:doi": "10.7927/H4F47M65", + "sci:citation": "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Population Density Adjusted to Match 2015 Revision UN WPP Country Totals, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H4F47M65. Accessed 22 October 2021.", + "file:size": 3031460, + "raster:bands": [ + { + "sampling": "area", + "data_type": "float32" + } + ], + "roles": ["data"] + } + }, + "bbox": [-180.0, 90.0, 180.0, -90.0], + "stac_extensions": [ + "https://stac-extensions.github.io/projection/v1.0.0/schema.json", + "https://stac-extensions.github.io/version/v1.0.0/schema.json", + "https://stac-extensions.github.io/file/v2.0.0/schema.json", + "https://stac-extensions.github.io/raster/v1.0.0/schema.json" + ] +} diff --git a/mypy.ini b/mypy.ini index 8cfa412..9ff707e 100644 --- a/mypy.ini +++ b/mypy.ini @@ -2,3 +2,27 @@ mypy_path = src explicit_package_bases = True namespace_packages = True + +check_untyped_defs = True +disallow_any_generics = True +disallow_incomplete_defs = True +disallow_subclassing_any = True +disallow_untyped_calls = False +disallow_untyped_decorators = False +disallow_untyped_defs = True +no_implicit_optional = True +show_error_codes = True +strict_equality = True +warn_redundant_casts = True +warn_return_any = True +warn_unused_configs = True +warn_unused_ignores = True + +[mypy-fsspec.*] +ignore_missing_imports = True + +[mypy-rasterio.*] +ignore_missing_imports = True + +[mypy-shapely.*] +ignore_missing_imports = True diff --git a/requirements-dev.txt b/requirements-dev.txt index 610192c..fe39c54 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,6 +2,7 @@ codespell coverage editorconfig-checker flake8 +isort jupyter mypy pylint diff --git a/scripts/format b/scripts/format index 07f36f7..9adfdf8 100755 --- a/scripts/format +++ b/scripts/format @@ -19,6 +19,8 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then if [ "${1:-}" = "--help" ]; then usage else + # Sort imports + isort --overwrite-in-place . # Code formatting yapf -ipr ${DIRS_TO_CHECK[@]} fi diff --git a/scripts/lint b/scripts/lint index e48036f..437f786 100755 --- a/scripts/lint +++ b/scripts/lint @@ -13,7 +13,7 @@ Execute project linters. " } -EC_EXCLUDE="(__pycache__|.git|.coverage|coverage.xml|.*\.egg-info|.mypy_cache|.tif|.tiff|.npy)" +EC_EXCLUDE="(__pycache__|.git|.coverage|coverage.xml|.*\.egg-info|.mypy_cache|.tif|.tiff|.npy|.ipynb)" DIRS_TO_CHECK=("src" "tests" "scripts") @@ -23,6 +23,8 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then else # Text formatting ec --exclude "$EC_EXCLUDE" + # Sort imports + isort --check . # Code formatting yapf -dpr ${DIRS_TO_CHECK[@]} # Lint diff --git a/setup.cfg b/setup.cfg index 066cdc3..a83f7fe 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,7 +27,9 @@ package_dir = = src packages = find_namespace: install_requires = - stactools == 0.2.1 + pytz == 2021.3 + stactools == 0.2.3 + types-python-dateutil == 2.8.2 [options.packages.find] where = src diff --git a/src/stactools/gpw/__init__.py b/src/stactools/gpw/__init__.py index d3ec452..2c43a82 100644 --- a/src/stactools/gpw/__init__.py +++ b/src/stactools/gpw/__init__.py @@ -1 +1,16 @@ +import stactools.core +from stactools.cli import Registry + +from stactools.gpw import commands +from stactools.gpw.stac import create_pop_collection, create_pop_item + +__all__ = ["create_pop_collection", "create_pop_item"] + +stactools.core.use_fsspec() + + +def register_plugin(registry: Registry) -> None: + registry.register_subcommand(commands.create_gpw_command) + + __version__ = "0.2.0" diff --git a/src/stactools/gpw/assets.py b/src/stactools/gpw/assets.py new file mode 100644 index 0000000..de87b13 --- /dev/null +++ b/src/stactools/gpw/assets.py @@ -0,0 +1,267 @@ +from typing import Dict + +import pystac +from pystac.extensions.item_assets import AssetDefinition + +POP_COUNT_KEY = "pop_count" +POP_COUNT_ADJ_KEY = "pop_count_adj" +POP_DENSITY_KEY = "pop_density" +POP_DENSITY_ADJ_KEY = "pop_density_adj" +ANC_DQI_CONTEXT_KEY = "anc_dqi_context" +ANC_DQI_ADMIN_KEY = "anc_dqi_admin" +ANC_DQI_WATERMASK_KEY = "anc_dqi_watermask" +ANC_BDC_BT_COUNT = "anc_bdc_bt_count" +ANC_BDC_BT_DENSITY = "anc_bdc_bt_density" +ANC_BDC_FT_COUNT = "anc_bdc_ft_count" +ANC_BDC_FT_DENSITY = "anc_bdc_ft_density" +ANC_BDC_MT_COUNT = "anc_bdc_mt_count" +ANC_BDC_MT_DENSITY = "anc_bdc_mt_density" +ANC_LAND_AREA_KEY = "anc_land_area" +ANC_WATER_AREA_KEY = "anc_water_area" +ANC_NAT_ID_GRID_KEY = "anc_nat_id_grid" + +POP_ITEM_ASSETS: Dict[str, AssetDefinition] = { + POP_COUNT_KEY: + AssetDefinition({ + "title": + "Population Count", + "description": ( + "Gridded Population of the World, Version 4 (GPWv4): Population Count, 30 arc-seconds, 1km resolution" # noqa: E501 + ), + "type": + pystac.MediaType.COG, + "roles": ["data"], + "sci:doi": + "10.7927/H4JW8BX5", + "sci:citation": + "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Population Count, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H4JW8BX5. Accessed 22 October 2021.", # noqa: E501 + }), + POP_COUNT_ADJ_KEY: + AssetDefinition({ + "title": + "UN WPP-Adjusted Population Count", + "description": ( + "Gridded Population of the World, Version 4 (GPWv4): UN WPP-Adjusted Population Count, 30 arc-seconds, 1km resolution" # noqa: E501 + ), + "type": + pystac.MediaType.COG, + "roles": ["data"], + "sci:doi": + "10.7927/H4PN93PB", + "sci:citation": + "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Population Count Adjusted to Match 2015 Revision of UN WPP Country Totals, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H4PN93PB. Accessed 22 October 2021.", # noqa: E501 + }), + POP_DENSITY_KEY: + AssetDefinition({ + "title": + "Population Density", + "description": ( + "Gridded Population of the World, Version 4 (GPWv4): Population Density, 30 arc-seconds, 1km resolution" # noqa: E501 + ), + "type": + pystac.MediaType.COG, + "roles": ["data"], + "sci:doi": + "10.7927/H49C6VHW", + "sci:citation": + "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Population Density, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H49C6VHW. Accessed 22 October 2021.", # noqa: E501 + }), + POP_DENSITY_ADJ_KEY: + AssetDefinition({ + "title": + "UN WPP-Adjusted Population Density", + "description": ( + "Gridded Population of the World, Version 4 (GPWv4): UN WPP-Adjusted Population Density, 30 arc-seconds, 1km resolution" # noqa: E501 + ), + "type": + pystac.MediaType.COG, + "roles": ["data"], + "sci:doi": + "10.7927/H4F47M65", + "sci:citation": + "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Population Density Adjusted to Match 2015 Revision UN WPP Country Totals, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H4F47M65. Accessed 22 October 2021.", # noqa: E501 + }), +} + +ANC_ITEM_ASSETS: Dict[str, AssetDefinition] = { + ANC_DQI_CONTEXT_KEY: + AssetDefinition({ + "title": + "Data Quality Indicators - Data Context", + "description": ( + "Gridded Population of the World, Version 4 (GPWv4): Data Quality Indicators - Data Context, 30 arc-seconds, 1km resolution" # noqa: E501 + ), + "type": + pystac.MediaType.COG, + "roles": ["data"], + "sci:doi": + "10.7927/H42Z13KG", + "sci:citation": + "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Data Quality Indicators, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H42Z13KG. Accessed 22 Octover 2021.", # noqa: E501 + }), + ANC_DQI_ADMIN_KEY: + AssetDefinition({ + "title": + "Data Quality Indicators - Mean Administrative Unit Area", + "description": ( + "Gridded Population of the World, Version 4 (GPWv4): Data Quality Indicators - Mean Administrative Unit Area, 30 arc-seconds, 1km resolution" # noqa: E501 + ), + "type": + pystac.MediaType.COG, + "roles": ["data"], + "sci:doi": + "10.7927/H42Z13KG", + "sci:citation": + "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Data Quality Indicators, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H42Z13KG. Accessed 22 October 2021.", # noqa: E501 + }), + ANC_DQI_WATERMASK_KEY: + AssetDefinition({ + "title": + "Data Quality Indicators - Water Mask", + "description": ( + "Gridded Population of the World, Version 4 (GPWv4): Data Quality Indicators - Water Mask, 30 arc-seconds, 1km resolution" # noqa: E501 + ), + "type": + pystac.MediaType.COG, + "roles": ["data"], + "sci:doi": + "10.7927/H42Z13KG", + "sci:citation": + "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Data Quality Indicators, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H42Z13KG. Accessed 22 October 2021.", # noqa: E501 + }), + ANC_BDC_BT_COUNT: + AssetDefinition({ + "title": + "Basic Demographic Characteristics - Total Population Count", + "description": ( + "Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics - Total Population Count, 30 arc-seconds, 1km resolution" # noqa: E501 + ), + "type": + pystac.MediaType.COG, + "roles": ["data"], + "sci:doi": + "10.7927/H46M34XX", + "sci:citation": + "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H46M34XX. Accessed 22 October 2021.", # noqa: E501 + }), + ANC_BDC_BT_DENSITY: + AssetDefinition({ + "title": + "Basic Demographic Characteristics - Total Population Density", + "description": ( + "Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics - Total Population Density, 30 arc-seconds, 1km resolution" # noqa: E501 + ), + "type": + pystac.MediaType.COG, + "roles": ["data"], + "sci:doi": + "10.7927/H46M34XX", + "sci:citation": + "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H46M34XX. Accessed 22 October 2021.", # noqa: E501 + }), + ANC_BDC_FT_COUNT: + AssetDefinition({ + "title": + "Basic Demographic Characteristics - Female Count", + "description": ( + "Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics - Female Count, 30 arc-seconds, 1km resolution" # noqa: E501 + ), + "type": + pystac.MediaType.COG, + "roles": ["data"], + "sci:doi": + "10.7927/H46M34XX", + "sci:citation": + "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H46M34XX. Accessed 22 October 2021.", # noqa: E501 + }), + ANC_BDC_FT_DENSITY: + AssetDefinition({ + "title": + "Basic Demographic Characteristics - Female Density", + "description": ( + "Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics - Female Density, 30 arc-seconds, 1km resolution" # noqa: E501 + ), + "type": + pystac.MediaType.COG, + "roles": ["data"], + "sci:doi": + "10.7927/H46M34XX", + "sci:citation": + "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H46M34XX. Accessed 22 October 2021.", # noqa: E501 + }), + ANC_BDC_MT_COUNT: + AssetDefinition({ + "title": + "Basic Demographic Characteristics - Male Count", + "description": ( + "Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics - Male Count, 30 arc-seconds, 1km resolution" # noqa: E501 + ), + "type": + pystac.MediaType.COG, + "roles": ["data"], + "sci:doi": + "10.7927/H46M34XX", + "sci:citation": + "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H46M34XX. Accessed 22 October 2021.", # noqa: E501 + }), + ANC_BDC_MT_DENSITY: + AssetDefinition({ + "title": + "Basic Demographic Characteristics - Male Density", + "description": ( + "Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics - Male Density, 30 arc-seconds, 1km resolution" # noqa: E501 + ), + "type": + pystac.MediaType.COG, + "roles": ["data"], + "sci:doi": + "10.7927/H46M34XX", + "sci:citation": + "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Basic Demographic Characteristics, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H46M34XX. Accessed 22 October 2021.", # noqa: E501 + }), + ANC_LAND_AREA_KEY: + AssetDefinition({ + "title": + "Land Area", + "description": ( + "Gridded Population of the World, Version 4 (GPWv4): Land Area, 30 arc-seconds, 1km resolution" # noqa: E501 + ), + "type": + pystac.MediaType.COG, + "roles": ["data"], + "sci:doi": + "10.7927/H4Z60M4Z", + "sci:citation": + "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Land and Water Area, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H4Z60M4Z. Accessed 22 October 2021.", # noqa: E501 + }), + ANC_WATER_AREA_KEY: + AssetDefinition({ + "title": + "Water Area", + "description": ( + "Gridded Population of the World, Version 4 (GPWv4): Water Area, 30 arc-seconds, 1km resolution" # noqa: E501 + ), + "type": + pystac.MediaType.COG, + "roles": ["data"], + "sci:doi": + "10.7927/H4Z60M4Z", + "sci:citation": + "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): Land and Water Area, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H4Z60M4Z. Accessed 22 October 2021.", # noqa: E501 + }), + ANC_NAT_ID_GRID_KEY: + AssetDefinition({ + "title": + "National Identifier Grid", + "description": ( + "Gridded Population of the World, Version 4 (GPWv4): UN WPP-Adjusted Population Density, 30 arc-seconds, 1km resolution" # noqa: E501 + ), + "type": + pystac.MediaType.COG, + "roles": ["data"], + "sci:doi": + "10.7927/H4TD9VDP", + "sci:citation": + "Center for International Earth Science Information Network - CIESIN - Columbia University. 2018. Gridded Population of the World, Version 4 (GPWv4): National Identifier Grid, Revision 11. Palisades, NY: NASA Socioeconomic Data and Applications Center (SEDAC). https://doi.org/10.7927/H4TD9VDP. Accessed 22 October 2021.", # noqa: E501 + }), +} diff --git a/src/stactools/gpw/cog.py b/src/stactools/gpw/cog.py new file mode 100644 index 0000000..44bc1e3 --- /dev/null +++ b/src/stactools/gpw/cog.py @@ -0,0 +1,243 @@ +import logging +import os +from glob import glob +from subprocess import CalledProcessError, check_output +from tempfile import TemporaryDirectory +from typing import List, Union + +import rasterio + +from stactools.gpw.constants import GPW_TILING_PIXEL_SIZE + +logger = logging.getLogger(__name__) + + +def create_cog( + input_path: str, + output_dir: str, + raise_on_fail: bool = True, + dry_run: bool = False, + tile: bool = False, + expanded_bbox: List[Union[float, int, str]] = [], +) -> str: + """Create COG from a tif + + Args: + input_path (str): Path to tif. + output_path (str): The path to which the COG will be written. + raise_on_fail (bool, optional): Whether to raise error on failure. + Defaults to True. + dry_run (bool, optional): Run without downloading tif, creating COG, + and writing COG. Defaults to False. + tile (bool, optional): Tile the tiff into many smaller files + expanded_bbox (list, optional): Expand input bounding box before processing. Bounding + box should be in the order: [xmin, ymin, xmax, ymax] + + Returns: + str: The path to the output COG(s) + """ + + try: + if dry_run: + logger.info( + "Would have downloaded TIF, created COG, and written COG") + else: + if tile: + return create_retiled_cogs(input_path, output_dir, + raise_on_fail, dry_run, + expanded_bbox) + else: + output_path = os.path.join( + output_dir, + os.path.basename(input_path)[:-4] + "_cog.tif") + create_single_cog(input_path, output_path, raise_on_fail, + dry_run) + + except Exception: + logger.error("Failed to process {}".format(output_dir)) + + if raise_on_fail: + raise + + return output_path + + +def create_single_cog( + input_path: str, + output_path: str, + raise_on_fail: bool = True, + dry_run: bool = False, +) -> str: + """Create COG from a TIFF + Args: + input_path (str): Path to the GeoTiff data. + output_path (str): The path to which the COG will be written. + raise_on_fail (bool, optional): Whether to raise error on failure. + Defaults to True. + dry_run (bool, optional): Run without downloading TIFF, creating COG, + and writing COG. Defaults to False. + Returns: + str: The path to the output COG. + """ + + output = None + try: + if dry_run: + logger.info("Would have read TIFF, created COG, and written COG") + else: + logger.info("Converting TIFF to COG") + logger.debug(f"input_path: {input_path}") + logger.debug(f"output_path: {output_path}") + cmd = [ + "gdal_translate", + "-of", + "COG", + "-co", + "NUM_THREADS=ALL_CPUS", + "-co", + "BLOCKSIZE=512", + "-co", + "COMPRESS=DEFLATE", + "-co", + "LEVEL=9", + "-co", + "PREDICTOR=YES", + "-co", + "OVERVIEWS=IGNORE_EXISTING", + input_path, + output_path, + ] + + try: + output = check_output(cmd) + except CalledProcessError as e: + output = e.output + raise + finally: + logger.info(f"output: {str(output)}") + + except Exception: + logger.error("Failed to process {}".format(output_path)) + + if raise_on_fail: + raise + + return output_path + + +def create_retiled_cogs( + input_path: str, + output_directory: str, + raise_on_fail: bool = True, + dry_run: bool = False, + expanded_bbox: List[Union[float, int, str]] = [], +) -> str: + """Split tiff into tiles and create COGs + Args: + input_path (str): Path to the GeoTiff data. + output_directory (str): The directory to which the COG will be written. + raise_on_fail (bool, optional): Whether to raise error on failure. + Defaults to True. + dry_run (bool, optional): Run without downloading tif, creating COG, + and writing COG. Defaults to False. + expanded_bbox (list, optional): Expand input bounding box before processing. Bounding + box should be in the order: [xmin, ymin, xmax, ymax] + Returns: + str: The path to the output COGs. + """ + output = None + try: + if dry_run: + logger.info( + "Would have split TIFF into tiles, created COGs, and written COGs" + ) + else: + logger.info("Retiling TIFF") + logger.debug(f"input_path: {input_path}") + logger.debug(f"output_directory: {output_directory}") + with TemporaryDirectory() as tmp_dir: + if expanded_bbox: + input_path = expand_bbox(input_path, tmp_dir, + expanded_bbox) + + cmd = [ + "gdal_retile.py", + "-ps", + str(GPW_TILING_PIXEL_SIZE[0]), + str(GPW_TILING_PIXEL_SIZE[1]), + "-targetDir", + tmp_dir, + input_path, + ] + try: + output = check_output(cmd) + except CalledProcessError as e: + output = e.output + raise + finally: + logger.info(f"output: {str(output)}") + file_names = glob(f"{tmp_dir}/*.tif") + for f in [ + i for i in file_names + if not i.endswith("_expanded.tif") + ]: + input_file = os.path.join(tmp_dir, f) + output_file = os.path.join( + output_directory, + os.path.basename(f).replace(".tif", "") + "_cog.tif", + ) + with rasterio.open(input_file, "r") as dataset: + contains_data = dataset.read().any() + # Exclude empty files + if contains_data: + logger.debug(f"Tile contains data: {input_file}") + create_single_cog(input_file, output_file, + raise_on_fail, dry_run) + else: + logger.debug(f"Ignoring empty tile: {input_file}") + + except Exception: + logger.error("Failed to process {}".format(input_path)) + + if raise_on_fail: + raise + + return output_directory + + +def expand_bbox( + input_path: str, + tmp_dir: str, + expanded_bbox: List[Union[float, int, str]] = [], +) -> str: + output_path = os.path.join( + tmp_dir, + f'{os.path.basename(input_path).replace(".tif", "_expanded.tif")}') + cmd = [ + "gdalwarp", + "-overwrite", + "-co", + "NUM_THREADS=ALL_CPUS", + "-co", + "COMPRESS=DEFLATE", + "-te", + str(expanded_bbox[0]), + str(expanded_bbox[1]), + str(expanded_bbox[2]), + str(expanded_bbox[3]), + "-ts", + "43200", + "21600", + input_path, + output_path, + ] + + try: + output = check_output(cmd) + except CalledProcessError as e: + output = e.output + raise + finally: + logger.info(f"output: {str(output)}") + + return output_path diff --git a/src/stactools/gpw/commands.py b/src/stactools/gpw/commands.py new file mode 100644 index 0000000..aee2d1e --- /dev/null +++ b/src/stactools/gpw/commands.py @@ -0,0 +1,232 @@ +import logging +import os +from typing import List, Tuple, Union + +import click + +from stactools.gpw import cog, stac + +logger = logging.getLogger(__name__) + + +def create_gpw_command(cli: click.Group) -> click.Command: + """Creates the gpw command line utility.""" + @cli.group( + "gpw", + short_help="Commands for working with GPW data", + ) + def gpw() -> None: + pass + + @gpw.command( + "create-cog", + short_help="Transform Geotiff to Cloud-Optimized Geotiff.", + ) + @click.option("-d", + "--destination", + required=True, + help="The output directory for the COG") + @click.option("-s", + "--source", + required=True, + help="Path to an input GeoTiff") + @click.option( + "-t", + "--tile", + help="Tile the tiff into many smaller files.", + is_flag=True, + default=False, + ) + @click.option( + "-e", + "--expanded_bbox", + nargs=4, + required=False, + help="Expand retiled bbox to: xmin ymin xmax ymax", + ) + def create_cog_command( + destination: str, + source: str, + tile: bool, + expanded_bbox: Tuple[Union[float, int, str]], + ) -> None: + """Generate a COG from a GeoTiff. The COG will be saved in the destination + with `_cog.tif` appended to the name. + + Args: + destination (str): Local directory to save output COGs + source (str, optional): An input GPW GeoTiff + tile (bool): Tile the tiff into many smaller files + """ + if not os.path.isdir(destination): + raise IOError(f'Destination folder "{destination}" not found') + + expanded_bbox_list: List[Union[float, int, str]] = list(expanded_bbox) + cog.create_cog(source, + destination, + tile=tile, + expanded_bbox=expanded_bbox_list) + + @gpw.command( + "create-pop-collection", + short_help="Creates a STAC collection from GPW population metadata", + ) + @click.option( + "-d", + "--destination", + required=True, + help="The output path for the STAC Collection json", + ) + def create_pop_collection_command(destination: str) -> None: + """Creates a STAC Collection from GPW population metadata + + Args: + destination (str): Path used to store the collection json + Returns: + Callable + """ + stac.create_pop_collection(destination) + + @gpw.command( + "create-anc-collection", + short_help="Creates a STAC collection from GPW ancillary metadata", + ) + @click.option( + "-d", + "--destination", + required=True, + help="The output path for the STAC Collection json", + ) + def create_anc_collection_command(destination: str) -> None: + """Creates a STAC Collection from GPW ancillary metadata + + Args: + destination (str): Path used to store the collection json + Returns: + Callable + """ + stac.create_anc_collection(destination) + + @gpw.command( + "create-pop-item", + short_help="Create a STAC item for population datasets", + ) + @click.option( + "-d", + "--destination", + required=True, + help="The output directory for the STAC json", + ) + @click.option( + "-c", + "--cogs", + nargs=4, + required=True, + help=""" + \b + COG hrefs for: + Population Count, + UN WPP-Adjusted Population Count, + Population Density, + UN WPP-Adjusted Population Density + """, + ) + def create_pop_item_command(destination: str, cogs: Tuple[str]) -> None: + """Generate a population STAC item using the metadata. + + Args: + destination (str): Local directory to save the STAC Item json + cog (str): location of COG assets for the item + """ + + (pop_count, pop_count_adj, pop_density, + pop_density_adj) = (i for i in cogs) + + item = stac.create_pop_item(pop_count, pop_count_adj, pop_density, + pop_density_adj) + + output_path = os.path.join(destination, item.id + ".json") + + item.set_self_href(output_path) + item.make_asset_hrefs_relative() + item.save_object() + item.validate() + + @gpw.command( + "create-anc-item", + short_help="Create a STAC item for ancillary datasets", + ) + @click.option( + "-d", + "--destination", + required=True, + help="The output directory for the STAC json", + ) + @click.option( + "-c", + "--cogs", + nargs=12, + required=True, + help=""" + \b + COG hrefs for: + Data Quality Indicators - Data Context, + Data Quality Indicators - Mean Administrative Unit Area, + Data Quality Indicators - Water Mask, + Basic Demographic Characteristics - Total Population Count, + Basic Demographic Characteristics - Total Population Density, + Basic Demographic Characteristics - Female Count, + Basic Demographic Characteristics - Female Density, + Basic Demographic Characteristics - Male Count, + Basic Demographic Characteristics - Male Density, + Land Area, + Water Area, + National Identifier Grid + """, + ) + def create_anc_item_command(destination: str, cogs: Tuple[str]) -> None: + """Generate a population STAC item using the metadata. + + Args: + destination (str): Local directory to save the STAC Item json + cog (str): location of COG assets for the item + """ + + ( + dqi_context, + dqi_admin, + dqi_watermask, + bdc_bt_count, + bdc_bt_density, + bdc_ft_count, + bdc_ft_density, + bdc_mt_count, + bdc_mt_density, + land_area, + water_area, + nat_id_grid, + ) = (i for i in cogs) + + item = stac.create_anc_item( + dqi_context, + dqi_admin, + dqi_watermask, + bdc_bt_count, + bdc_bt_density, + bdc_ft_count, + bdc_ft_density, + bdc_mt_count, + bdc_mt_density, + land_area, + water_area, + nat_id_grid, + ) + + output_path = os.path.join(destination, item.id + ".json") + + item.set_self_href(output_path) + item.make_asset_hrefs_relative() + item.save_object() + item.validate() + + return gpw diff --git a/src/stactools/gpw/constants.py b/src/stactools/gpw/constants.py new file mode 100644 index 0000000..942d6e4 --- /dev/null +++ b/src/stactools/gpw/constants.py @@ -0,0 +1,47 @@ +# flake8: noqa + +from pyproj import CRS +from pystac import Link, Provider, ProviderRole + +GPW_EPSG = 4326 +GPW_CRS = CRS.from_epsg(GPW_EPSG) +GPW_LICENSE = "CC-BY-4.0" +GPW_LICENSE_LINK = Link( + rel="license", + target="https://creativecommons.org/licenses/by/4.0/legalcode", + title="Creative Commons Attribution 4.0 International", +) +GPW_PROVIDER = Provider( + name= + "Center for International Earth Science Information Network - CIESIN - Columbia University", + roles=[ProviderRole.PRODUCER, ProviderRole.PROCESSOR, ProviderRole.HOST], + url= + "https://sedac.ciesin.columbia.edu/data/set/gpw-v4-population-count-rev11", +) +GPW_BOUNDING_BOX = [-180.000000, 90.000000, 180.000000, -90.000000] +GPW_TILING_PIXEL_SIZE = (10001, 10001) +GPW_VERSION = "4.11" + +GPW_POP_ID = "GPW-population" +GPW_POP_TITLE = "Gridded Population of the World (GPW) v4.11, Population Datasets" +GPW_POP_DESCRIPTION = ( + f"The Gridded Population of the World, Version 4 (GPWv4) Revision 11 Population Datasets (population count, " + f"UN WPP-adjusted population count, population density, and UN WPP-adjusted population density) consist " + f"of estimates of human population (number of persons per pixel) and population density (number of persons " + f"per square kilometer), consistent with national censuses and population registers, for the years 2000, 2005, " + f"2010, 2015, and 2020. Where indicated by name, values have been adjusted to match the 2015 Revision of the " + f"United Nation's World Population Prospects (UN WPP) country totals.") + +GPW_POP_START_YEAR = "2000" +GPW_POP_END_YEAR = "2020" + +GPW_ANC_ID = "GPW-ancillary" +GPW_ANC_TITLE = "Gridded Population of the World (GPW) v4.11, Ancillary Datasets" +GPW_ANC_DESCRIPTION = ( + f"The Gridded Population of the World, Version 4 (GPWv4) Revision 11 Ancillary Datasets (data quality indicators, " + f"basic demographic characteristics, land and water area, and national identifier grid) provide context for the " + f"population count and density rasters found within the Gridded Population of the World Population Collection." +) + +GPW_ANC_START_YEAR = "2010" +GPW_ANC_END_YEAR = "2011" diff --git a/src/stactools/gpw/stac.py b/src/stactools/gpw/stac.py new file mode 100644 index 0000000..2f8a8ef --- /dev/null +++ b/src/stactools/gpw/stac.py @@ -0,0 +1,361 @@ +import logging +import os +import re +from datetime import datetime +from typing import Optional + +import fsspec +import pystac +import pytz +import rasterio +import shapely +from dateutil.relativedelta import relativedelta +from pystac.extensions.file import FileExtension +from pystac.extensions.item_assets import ItemAssetsExtension +from pystac.extensions.projection import ProjectionExtension +from pystac.extensions.raster import RasterBand, RasterExtension +from pystac.extensions.scientific import ScientificExtension +from pystac.extensions.version import ItemVersionExtension +from stactools.core.io import ReadHrefModifier + +from stactools.gpw.assets import ( + ANC_BDC_BT_COUNT, + ANC_BDC_BT_DENSITY, + ANC_BDC_FT_COUNT, + ANC_BDC_FT_DENSITY, + ANC_BDC_MT_COUNT, + ANC_BDC_MT_DENSITY, + ANC_DQI_ADMIN_KEY, + ANC_DQI_CONTEXT_KEY, + ANC_DQI_WATERMASK_KEY, + ANC_ITEM_ASSETS, + ANC_LAND_AREA_KEY, + ANC_NAT_ID_GRID_KEY, + ANC_WATER_AREA_KEY, + POP_COUNT_ADJ_KEY, + POP_COUNT_KEY, + POP_DENSITY_ADJ_KEY, + POP_DENSITY_KEY, + POP_ITEM_ASSETS, +) +from stactools.gpw.constants import ( + GPW_ANC_DESCRIPTION, + GPW_ANC_END_YEAR, + GPW_ANC_ID, + GPW_ANC_START_YEAR, + GPW_ANC_TITLE, + GPW_BOUNDING_BOX, + GPW_EPSG, + GPW_LICENSE, + GPW_LICENSE_LINK, + GPW_POP_DESCRIPTION, + GPW_POP_END_YEAR, + GPW_POP_ID, + GPW_POP_START_YEAR, + GPW_POP_TITLE, + GPW_PROVIDER, + GPW_VERSION, +) + +logger = logging.getLogger(__name__) + + +def create_pop_item( + pop_count: str, + pop_count_adj: str, + pop_density: str, + pop_density_adj: str, + cog_href_modifier: Optional[ReadHrefModifier] = None, +) -> pystac.Item: + """Creates a STAC item for Gridded Population of the World, + Version 4 (GPWv4): Population datasets. + + Args: + pop_count (str): Path to tiled population count COG + pop_count_adj (str): Path to tiled adjusted population count COG + pop_density (str): Path to tiled population density COG + pop_density_adj (str): Path to adjusted tiled population density COG + + Returns: + pystac.Item: STAC Item object. + """ + + # construct an item id like: gpw_v4_rev11_2005_30_sec_1_1 + split_basename = os.path.basename(pop_count).split("_") + item_id = "_".join([*split_basename[0:2], *split_basename[4:10]]) + + match = re.search(r"\d{4}", item_id) + assert match + year = match.group() + + utc = pytz.utc + + dataset_datetime = utc.localize(datetime.strptime(year, "%Y")) + + end_datetime = dataset_datetime + relativedelta(years=5) + + start_datetime = dataset_datetime + end_datetime = end_datetime + + polygon = shapely.geometry.box(*GPW_BOUNDING_BOX, ccw=True) + coordinates = [list(i) for i in list(polygon.exterior.coords)] + + geometry = {"type": "Polygon", "coordinates": [coordinates]} + + properties = { + "description": GPW_POP_DESCRIPTION, + } + + # Create item + item = pystac.Item( + id=item_id, + geometry=geometry, + bbox=GPW_BOUNDING_BOX, + datetime=dataset_datetime, + properties=properties, + ) + + item.common_metadata.start_datetime = start_datetime + item.common_metadata.end_datetime = end_datetime + + item_projection = ProjectionExtension.ext(item, add_if_missing=True) + item_projection.epsg = GPW_EPSG + item_projection.bbox = GPW_BOUNDING_BOX + + signed_cog_href = cog_href_modifier( + pop_count) if cog_href_modifier else pop_count + + src = rasterio.open(signed_cog_href) + + item_projection.transform = list(src.transform) + item_projection.shape = [src.height, src.width] + + item_version = ItemVersionExtension.ext(item, add_if_missing=True) + item_version.version = GPW_VERSION + + for key, href in [ + (POP_COUNT_KEY, pop_count), + (POP_COUNT_ADJ_KEY, pop_count_adj), + (POP_DENSITY_KEY, pop_density), + (POP_DENSITY_ADJ_KEY, pop_density_adj), + ]: + cog_asset = POP_ITEM_ASSETS[key].create_asset(href) + item.add_asset(key, cog_asset) + + signed_cog_href = cog_href_modifier( + href) if cog_href_modifier else href + + asset_file = FileExtension.ext(cog_asset, add_if_missing=True) + with fsspec.open(signed_cog_href) as file: + size = file.size + if size is not None: + asset_file.size = size + + with rasterio.open(signed_cog_href) as src: + asset_raster = RasterExtension.ext(cog_asset, add_if_missing=True) + asset_raster.bands = [ + RasterBand.create( + data_type=src.dtypes[0], + sampling=src.tags().get("AREA_OR_POINT").lower(), + ) + ] + + return item + + +def create_anc_item( + dqi_context: str, + dqi_admin: str, + dqi_watermask: str, + bdc_bt_count: str, + bdc_bt_density: str, + bdc_ft_count: str, + bdc_ft_density: str, + bdc_mt_count: str, + bdc_mt_density: str, + land_area: str, + water_area: str, + nat_id_grid: str, + cog_href_modifier: Optional[ReadHrefModifier] = None, +) -> pystac.Item: + """Creates a STAC item for Gridded Population of the World, + Version 4 (GPWv4): Ancillary datasets. + + Args: + dqi_context (str): Path to tiled Data Quality Indicators - Data Context COG + dqi_admin (str): Path to tiled Data Quality Indicators - Mean Administrative Unit Area COG + dqi_watermask (str): Path to tiled Data Quality Indicators - Water Mask COG + bdc_bt_count (str): Path to the tiled Basic Demographic Characteristics - Ttl Pop Count + bdc_bt_density (str): Path to the tiled Basic Demographic Characteristics - Ttl Pop Density + bdc_ft_count (str): Path to the tiled Basic Demographic Characteristics - Female Count + bdc_ft_density (str): Path to the tiled Basic Demographic Characteristics - Female Density + bdc_mt_count (str): Path to the tiled Basic Demographic Characteristics - Male Count + bdc_mt_density (str): Path to the tiled Basic Demographic Characteristics - Male Density + land_area (str): Path to tiled Land Area COG + water_area (str): Path to tiled Water Area COG + nat_id_grid (str): Path to tiled National Identifier Grid COG + cog_href_modifier (Optional[ReadHrefModifier], optional): [description]. Defaults to None. + + Returns: + pystac.Item: STAC Item object. + """ + + # construct an item id like: gpw_v4_rev11_30_sec_1_1 + split_basename = os.path.basename(dqi_context).split("_") + item_id = "_".join( + [*split_basename[0:2], split_basename[5], *split_basename[7:11]]) + + utc = pytz.utc + + dataset_datetime = utc.localize(datetime.strptime("2010-07-01", + "%Y-%m-%d")) + + polygon = shapely.geometry.box(*GPW_BOUNDING_BOX, ccw=True) + coordinates = [list(i) for i in list(polygon.exterior.coords)] + + geometry = {"type": "Polygon", "coordinates": [coordinates]} + + properties = { + "description": GPW_ANC_DESCRIPTION, + } + + # Create item + item = pystac.Item( + id=item_id, + geometry=geometry, + bbox=GPW_BOUNDING_BOX, + datetime=dataset_datetime, + properties=properties, + ) + + item_projection = ProjectionExtension.ext(item, add_if_missing=True) + item_projection.epsg = GPW_EPSG + item_projection.bbox = GPW_BOUNDING_BOX + + signed_cog_href = (cog_href_modifier(dqi_context) + if cog_href_modifier else dqi_context) + + src = rasterio.open(signed_cog_href) + + item_projection.transform = list(src.transform) + item_projection.shape = [src.height, src.width] + + item_version = ItemVersionExtension.ext(item, add_if_missing=True) + item_version.version = GPW_VERSION + + for key, href in [ + (ANC_DQI_CONTEXT_KEY, dqi_context), + (ANC_DQI_ADMIN_KEY, dqi_admin), + (ANC_DQI_WATERMASK_KEY, dqi_watermask), + (ANC_BDC_BT_COUNT, bdc_bt_count), + (ANC_BDC_BT_DENSITY, bdc_bt_density), + (ANC_BDC_FT_COUNT, bdc_ft_count), + (ANC_BDC_FT_DENSITY, bdc_ft_density), + (ANC_BDC_MT_COUNT, bdc_mt_count), + (ANC_BDC_MT_DENSITY, bdc_mt_density), + (ANC_LAND_AREA_KEY, land_area), + (ANC_WATER_AREA_KEY, water_area), + (ANC_NAT_ID_GRID_KEY, nat_id_grid), + ]: + cog_asset = ANC_ITEM_ASSETS[key].create_asset(href) + item.add_asset(key, cog_asset) + + signed_cog_href = cog_href_modifier( + href) if cog_href_modifier else href + + asset_file = FileExtension.ext(cog_asset, add_if_missing=True) + with fsspec.open(signed_cog_href) as file: + size = file.size + if size is not None: + asset_file.size = size + + with rasterio.open(signed_cog_href) as src: + asset_raster = RasterExtension.ext(cog_asset, add_if_missing=True) + asset_raster.bands = [ + RasterBand.create( + data_type=src.dtypes[0], + sampling=src.tags().get("AREA_OR_POINT").lower(), + ) + ] + + return item + + +def create_pop_collection(output_url: str) -> pystac.Collection: + """Create a STAC Collection for Gridded Population of + the World, Version 4 (GPWv4): Population datasets + + Args: + output (str): Location to save the output STAC Collection json + + Returns: + pystac.Collection: pystac collection object + """ + utc = pytz.utc + + start_datetime = utc.localize(datetime.strptime(GPW_POP_START_YEAR, "%Y")) + end_datetime = utc.localize(datetime.strptime(GPW_POP_END_YEAR, "%Y")) + + collection = pystac.Collection( + id=GPW_POP_ID, + title=GPW_POP_TITLE, + description=GPW_POP_DESCRIPTION, + providers=[GPW_PROVIDER], + license=GPW_LICENSE, + extent=pystac.Extent( + pystac.SpatialExtent(GPW_BOUNDING_BOX), + # `or None` fixes mypy issue with the DateTime being non-Optional + pystac.TemporalExtent([[start_datetime or None, end_datetime]]), + ), + catalog_type=pystac.CatalogType.RELATIVE_PUBLISHED, + ) + collection.add_link(GPW_LICENSE_LINK) + item_assets = ItemAssetsExtension.ext(collection, add_if_missing=True) + item_assets.item_assets = POP_ITEM_ASSETS + + ScientificExtension.ext(collection, add_if_missing=True) + + collection.set_self_href(output_url) + collection.save_object() + + return collection + + +def create_anc_collection(output_url: str) -> pystac.Collection: + """Create a STAC Collection for Gridded Population of + the World, Version 4 (GPWv4): Ancillary datasets + + Args: + output (str): Location to save the output STAC Collection json + + Returns: + pystac.Collection: pystac collection object + """ + utc = pytz.utc + + start_datetime = utc.localize(datetime.strptime(GPW_ANC_START_YEAR, "%Y")) + end_datetime = utc.localize(datetime.strptime(GPW_ANC_END_YEAR, "%Y")) + + collection = pystac.Collection( + id=GPW_ANC_ID, + title=GPW_ANC_TITLE, + description=GPW_ANC_DESCRIPTION, + providers=[GPW_PROVIDER], + license=GPW_LICENSE, + extent=pystac.Extent( + pystac.SpatialExtent(GPW_BOUNDING_BOX), + # `or None` fixes mypy issue with the DateTime being non-Optional + pystac.TemporalExtent([[start_datetime or None, end_datetime]]), + ), + catalog_type=pystac.CatalogType.RELATIVE_PUBLISHED, + ) + collection.add_link(GPW_LICENSE_LINK) + item_assets = ItemAssetsExtension.ext(collection, add_if_missing=True) + item_assets.item_assets = ANC_ITEM_ASSETS + + ScientificExtension.ext(collection, add_if_missing=True) + + collection.set_self_href(output_url) + collection.save_object() + + return collection diff --git a/tests/__init__.py b/tests/__init__.py index e69de29..5067606 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +from stactools.testing import TestData + +test_data = TestData(__file__) diff --git a/tests/data-files/raw/population/gpw_v4_population_count_rev11_2000_30_sec.tif b/tests/data-files/raw/population/gpw_v4_population_count_rev11_2000_30_sec.tif new file mode 100644 index 0000000..891e520 Binary files /dev/null and b/tests/data-files/raw/population/gpw_v4_population_count_rev11_2000_30_sec.tif differ diff --git a/tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopbt_2010_cntm_30_sec_2_1_cog.tif b/tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopbt_2010_cntm_30_sec_2_1_cog.tif new file mode 100644 index 0000000..248e771 Binary files /dev/null and b/tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopbt_2010_cntm_30_sec_2_1_cog.tif differ diff --git a/tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopbt_2010_dens_30_sec_2_1_cog.tif b/tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopbt_2010_dens_30_sec_2_1_cog.tif new file mode 100644 index 0000000..e01816d Binary files /dev/null and b/tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopbt_2010_dens_30_sec_2_1_cog.tif differ diff --git a/tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopft_2010_cntm_30_sec_2_1_cog.tif b/tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopft_2010_cntm_30_sec_2_1_cog.tif new file mode 100644 index 0000000..4d15b69 Binary files /dev/null and b/tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopft_2010_cntm_30_sec_2_1_cog.tif differ diff --git a/tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopft_2010_dens_30_sec_2_1_cog.tif b/tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopft_2010_dens_30_sec_2_1_cog.tif new file mode 100644 index 0000000..516737d Binary files /dev/null and b/tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopft_2010_dens_30_sec_2_1_cog.tif differ diff --git a/tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopmt_2010_cntm_30_sec_2_1_cog.tif b/tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopmt_2010_cntm_30_sec_2_1_cog.tif new file mode 100644 index 0000000..728fcf8 Binary files /dev/null and b/tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopmt_2010_cntm_30_sec_2_1_cog.tif differ diff --git a/tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopmt_2010_dens_30_sec_2_1_cog.tif b/tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopmt_2010_dens_30_sec_2_1_cog.tif new file mode 100644 index 0000000..11e7947 Binary files /dev/null and b/tests/data-files/tiles/ancillary/gpw_v4_basic_demographic_characteristics_rev11_atotpopmt_2010_dens_30_sec_2_1_cog.tif differ diff --git a/tests/data-files/tiles/ancillary/gpw_v4_data_quality_indicators_rev11_context_30_sec_2_1_cog.tif b/tests/data-files/tiles/ancillary/gpw_v4_data_quality_indicators_rev11_context_30_sec_2_1_cog.tif new file mode 100644 index 0000000..fca4817 Binary files /dev/null and b/tests/data-files/tiles/ancillary/gpw_v4_data_quality_indicators_rev11_context_30_sec_2_1_cog.tif differ diff --git a/tests/data-files/tiles/ancillary/gpw_v4_data_quality_indicators_rev11_mean_adminunitarea_30_sec_2_1_cog.tif b/tests/data-files/tiles/ancillary/gpw_v4_data_quality_indicators_rev11_mean_adminunitarea_30_sec_2_1_cog.tif new file mode 100644 index 0000000..2eb5005 Binary files /dev/null and b/tests/data-files/tiles/ancillary/gpw_v4_data_quality_indicators_rev11_mean_adminunitarea_30_sec_2_1_cog.tif differ diff --git a/tests/data-files/tiles/ancillary/gpw_v4_data_quality_indicators_rev11_watermask_30_sec_2_1_cog.tif b/tests/data-files/tiles/ancillary/gpw_v4_data_quality_indicators_rev11_watermask_30_sec_2_1_cog.tif new file mode 100644 index 0000000..0833d14 Binary files /dev/null and b/tests/data-files/tiles/ancillary/gpw_v4_data_quality_indicators_rev11_watermask_30_sec_2_1_cog.tif differ diff --git a/tests/data-files/tiles/ancillary/gpw_v4_land_water_area_rev11_landareakm_30_sec_2_1_cog.tif b/tests/data-files/tiles/ancillary/gpw_v4_land_water_area_rev11_landareakm_30_sec_2_1_cog.tif new file mode 100644 index 0000000..9941c3a Binary files /dev/null and b/tests/data-files/tiles/ancillary/gpw_v4_land_water_area_rev11_landareakm_30_sec_2_1_cog.tif differ diff --git a/tests/data-files/tiles/ancillary/gpw_v4_land_water_area_rev11_waterareakm_30_sec_expanded_2_1_cog.tif b/tests/data-files/tiles/ancillary/gpw_v4_land_water_area_rev11_waterareakm_30_sec_expanded_2_1_cog.tif new file mode 100644 index 0000000..0e80ef7 Binary files /dev/null and b/tests/data-files/tiles/ancillary/gpw_v4_land_water_area_rev11_waterareakm_30_sec_expanded_2_1_cog.tif differ diff --git a/tests/data-files/tiles/ancillary/gpw_v4_national_identifier_grid_rev11_30_sec_2_1_cog.tif b/tests/data-files/tiles/ancillary/gpw_v4_national_identifier_grid_rev11_30_sec_2_1_cog.tif new file mode 100644 index 0000000..185fe30 Binary files /dev/null and b/tests/data-files/tiles/ancillary/gpw_v4_national_identifier_grid_rev11_30_sec_2_1_cog.tif differ diff --git a/tests/data-files/tiles/population/gpw_v4_population_count_adjusted_to_2015_unwpp_country_totals_rev11_2000_30_sec_2_1_cog.tif b/tests/data-files/tiles/population/gpw_v4_population_count_adjusted_to_2015_unwpp_country_totals_rev11_2000_30_sec_2_1_cog.tif new file mode 100644 index 0000000..c7d5646 Binary files /dev/null and b/tests/data-files/tiles/population/gpw_v4_population_count_adjusted_to_2015_unwpp_country_totals_rev11_2000_30_sec_2_1_cog.tif differ diff --git a/tests/data-files/tiles/population/gpw_v4_population_count_rev11_2000_30_sec_2_1_cog.tif b/tests/data-files/tiles/population/gpw_v4_population_count_rev11_2000_30_sec_2_1_cog.tif new file mode 100644 index 0000000..d3907e4 Binary files /dev/null and b/tests/data-files/tiles/population/gpw_v4_population_count_rev11_2000_30_sec_2_1_cog.tif differ diff --git a/tests/data-files/tiles/population/gpw_v4_population_density_adjusted_to_2015_unwpp_country_totals_rev11_2000_30_sec_2_1_cog.tif b/tests/data-files/tiles/population/gpw_v4_population_density_adjusted_to_2015_unwpp_country_totals_rev11_2000_30_sec_2_1_cog.tif new file mode 100644 index 0000000..328fc17 Binary files /dev/null and b/tests/data-files/tiles/population/gpw_v4_population_density_adjusted_to_2015_unwpp_country_totals_rev11_2000_30_sec_2_1_cog.tif differ diff --git a/tests/data-files/tiles/population/gpw_v4_population_density_rev11_2000_30_sec_2_1_cog.tif b/tests/data-files/tiles/population/gpw_v4_population_density_rev11_2000_30_sec_2_1_cog.tif new file mode 100644 index 0000000..2e59411 Binary files /dev/null and b/tests/data-files/tiles/population/gpw_v4_population_density_rev11_2000_30_sec_2_1_cog.tif differ diff --git a/tests/test_commands.py b/tests/test_commands.py new file mode 100644 index 0000000..3ad6d20 --- /dev/null +++ b/tests/test_commands.py @@ -0,0 +1,115 @@ +import logging +import os.path +from tempfile import TemporaryDirectory + +import pystac +from stactools.testing import CliTestCase + +from stactools.gpw.commands import create_gpw_command +from tests import test_data + +logger = logging.getLogger(__name__) + + +class CreateCollectionTest(CliTestCase): + def create_subcommand_functions(self): + return [create_gpw_command] + + def test_create_pop_collection(self): + with TemporaryDirectory() as tmp_dir: + json_path = os.path.join(tmp_dir, "test.json") + result = self.run_command( + ["gpw", "create-pop-collection", "-d", json_path]) + self.assertEqual(result.exit_code, + 0, + msg="\n{}".format(result.output)) + + jsons = [p for p in os.listdir(tmp_dir) if p.endswith(".json")] + self.assertEqual(len(jsons), 1) + + collection = pystac.read_file(os.path.join(tmp_dir, jsons[0])) + + collection.validate() + + def test_create_anc_collection(self): + with TemporaryDirectory() as tmp_dir: + json_path = os.path.join(tmp_dir, "test.json") + result = self.run_command( + ["gpw", "create-anc-collection", "-d", json_path]) + self.assertEqual(result.exit_code, + 0, + msg="\n{}".format(result.output)) + + jsons = [p for p in os.listdir(tmp_dir) if p.endswith(".json")] + self.assertEqual(len(jsons), 1) + + collection = pystac.read_file(os.path.join(tmp_dir, jsons[0])) + + collection.validate() + + def test_create_cog(self): + with TemporaryDirectory() as tmp_dir: + test_path = test_data.get_path("data-files/raw/population") + paths = [ + os.path.join(test_path, d) for d in os.listdir(test_path) + if d.lower().endswith(".tif") + ] + + for path in paths: + result = self.run_command( + ["gpw", "create-cog", "-d", tmp_dir, "-s", path]) + self.assertEqual(result.exit_code, + 0, + msg="\n{}".format(result.output)) + + cogs = [p for p in os.listdir(tmp_dir) if p.endswith("_cog.tif")] + + self.assertEqual(len(cogs), 1) + + def test_create_pop_item(self): + with TemporaryDirectory() as tmp_dir: + + test_path = test_data.get_path("data-files/tiles/population") + paths = [ + os.path.join(test_path, d) for d in os.listdir(test_path) + if d.lower().endswith(".tif") + ] + + result = self.run_command( + ["gpw", "create-pop-item", "-d", tmp_dir, "-c"] + paths) + self.assertEqual(result.exit_code, + 0, + msg="\n{}".format(result.output)) + + jsons = [p for p in os.listdir(tmp_dir) if p.endswith(".json")] + self.assertEqual(len(jsons), 1) + + item_path = os.path.join(tmp_dir, jsons[0]) + + item = pystac.read_file(item_path) + + item.validate() + + def test_create_anc_item(self): + with TemporaryDirectory() as tmp_dir: + + test_path = test_data.get_path("data-files/tiles/ancillary") + paths = [ + os.path.join(test_path, d) for d in os.listdir(test_path) + if d.lower().endswith(".tif") + ] + + result = self.run_command( + ["gpw", "create-anc-item", "-d", tmp_dir, "-c"] + paths) + self.assertEqual(result.exit_code, + 0, + msg="\n{}".format(result.output)) + + jsons = [p for p in os.listdir(tmp_dir) if p.endswith(".json")] + self.assertEqual(len(jsons), 1) + + item_path = os.path.join(tmp_dir, jsons[0]) + + item = pystac.read_file(item_path) + + item.validate() diff --git a/tests/test_stac.py b/tests/test_stac.py new file mode 100644 index 0000000..26e840d --- /dev/null +++ b/tests/test_stac.py @@ -0,0 +1,195 @@ +import os +import unittest +from tempfile import TemporaryDirectory + +import pystac +import rasterio + +from stactools.gpw import cog, stac +from tests import test_data + + +class StacTest(unittest.TestCase): + def test_create_cog(self): + with TemporaryDirectory() as tmp_dir: + test_path = test_data.get_path("data-files/raw/population") + paths = [ + os.path.join(test_path, d) for d in os.listdir(test_path) + if d.lower().endswith(".tif") + ] + + for path in paths: + cog.create_cog(path, tmp_dir) + + cogs = [p for p in os.listdir(tmp_dir) if p.endswith("_cog.tif")] + + self.assertEqual(len(cogs), 1) + + def test_create_tiled_cog(self): + with TemporaryDirectory() as tmp_dir: + test_path = test_data.get_path("data-files/raw/population") + paths = [ + os.path.join(test_path, d) for d in os.listdir(test_path) + if d.lower().endswith(".tif") + ] + + for path in paths: + output_path = os.path.join(tmp_dir) + cog.create_cog(path, output_path, tile=True) + + cogs = [p for p in os.listdir(tmp_dir) if p.endswith("_cog.tif")] + + self.assertEqual(len(cogs), 1) + + def test_create_tiled_cog_expand_bbox(self): + with TemporaryDirectory() as tmp_dir: + test_path = test_data.get_path( + "data-files/tiles/ancillary/gpw_v4_data_quality_indicators_rev11_context_30_sec_2_1_cog.tif" # noqa: E501 + ) + + expanded_bbox = [-180, -77, -95, 7] + + output_path = cog.expand_bbox(test_path, + tmp_dir, + expanded_bbox=expanded_bbox) + cogs = [ + p for p in os.listdir(tmp_dir) if p.endswith("expanded.tif") + ] + self.assertEqual(len(cogs), 1) + + with rasterio.open(os.path.join(tmp_dir, output_path)) as src: + self.assertEqual(int(src.bounds.left), int(expanded_bbox[0])) + self.assertEqual(int(src.bounds.bottom), int(expanded_bbox[1])) + self.assertEqual(int(src.bounds.right), int(expanded_bbox[2])) + self.assertEqual(int(src.bounds.top), int(expanded_bbox[3])) + + def test_create_pop_item(self): + with TemporaryDirectory() as tmp_dir: + + test_path = test_data.get_path("data-files/tiles/population") + paths = [ + os.path.join(test_path, d) for d in os.listdir(test_path) + if d.lower().endswith(".tif") + ] + + item = stac.create_pop_item(*paths) + json_path = os.path.join(tmp_dir, f"{item.id}.json") + item.set_self_href(json_path) + item.make_asset_hrefs_relative() + item.save_object() + + jsons = [p for p in os.listdir(tmp_dir) if p.endswith(".json")] + + self.assertEqual(len(jsons), 1) + + item_path = os.path.join(tmp_dir, jsons[0]) + + item = pystac.read_file(item_path) + + item.validate() + + def test_create_anc_item(self): + with TemporaryDirectory() as tmp_dir: + + test_path = test_data.get_path("data-files/tiles/ancillary") + + paths = [ + os.path.join( + test_path, + "gpw_v4_data_quality_indicators_rev11_context_30_sec_2_1_cog.tif", + ), + os.path.join( + test_path, + "gpw_v4_data_quality_indicators_rev11_mean_adminunitarea_30_sec_2_1_cog.tif", + ), + os.path.join( + test_path, + "gpw_v4_data_quality_indicators_rev11_watermask_30_sec_2_1_cog.tif", + ), + os.path.join( + test_path, + "gpw_v4_basic_demographic_characteristics_rev11_atotpopbt_2010_cntm_30_sec_2_1_cog.tif", # noqa:E501 + ), + os.path.join( + test_path, + "gpw_v4_basic_demographic_characteristics_rev11_atotpopbt_2010_dens_30_sec_2_1_cog.tif", # noqa:E501 + ), + os.path.join( + test_path, + "gpw_v4_basic_demographic_characteristics_rev11_atotpopft_2010_cntm_30_sec_2_1_cog.tif", # noqa:E501 + ), + os.path.join( + test_path, + "gpw_v4_basic_demographic_characteristics_rev11_atotpopft_2010_dens_30_sec_2_1_cog.tif", # noqa:E501 + ), + os.path.join( + test_path, + "gpw_v4_basic_demographic_characteristics_rev11_atotpopmt_2010_cntm_30_sec_2_1_cog.tif", # noqa:E501 + ), + os.path.join( + test_path, + "gpw_v4_basic_demographic_characteristics_rev11_atotpopmt_2010_dens_30_sec_2_1_cog.tif", # noqa:E501 + ), + os.path.join( + test_path, + "gpw_v4_land_water_area_rev11_landareakm_30_sec_2_1_cog.tif", + ), + os.path.join( + test_path, + "gpw_v4_land_water_area_rev11_waterareakm_30_sec_expanded_2_1_cog.tif", + ), + os.path.join( + test_path, + "gpw_v4_national_identifier_grid_rev11_30_sec_2_1_cog.tif", + ), + ] + + item = stac.create_anc_item(*paths) + json_path = os.path.join(tmp_dir, f"{item.id}.json") + item.set_self_href(json_path) + item.make_asset_hrefs_relative() + item.save_object() + + jsons = [p for p in os.listdir(tmp_dir) if p.endswith(".json")] + + self.assertEqual(len(jsons), 1) + + item_path = os.path.join(tmp_dir, jsons[0]) + + item = pystac.read_file(item_path) + + item.validate() + + def test_create_pop_collection(self): + with TemporaryDirectory() as tmp_dir: + + # Create stac collection + json_path = os.path.join(tmp_dir, "collection.json") + + stac.create_pop_collection(json_path) + + jsons = [p for p in os.listdir(tmp_dir) if p.endswith(".json")] + self.assertEqual(len(jsons), 1) + + collection_path = os.path.join(tmp_dir, jsons[0]) + + collection = pystac.read_file(collection_path) + + collection.validate() + + def test_create_anc_collection(self): + with TemporaryDirectory() as tmp_dir: + + # Create stac collection + json_path = os.path.join(tmp_dir, "collection.json") + + stac.create_anc_collection(json_path) + + jsons = [p for p in os.listdir(tmp_dir) if p.endswith(".json")] + self.assertEqual(len(jsons), 1) + + collection_path = os.path.join(tmp_dir, jsons[0]) + + collection = pystac.read_file(collection_path) + + collection.validate()