From 8a22a34d9998545058883360e61ada5ba3a3c7a3 Mon Sep 17 00:00:00 2001 From: Simon Norris Date: Wed, 16 Jul 2025 12:23:58 -0700 Subject: [PATCH 1/2] remove WCS tooling --- README.md | 28 +-------- scripts/document_cli.sh | 6 -- src/bcdata/__init__.py | 1 - src/bcdata/cli.py | 53 ---------------- src/bcdata/wcs.py | 130 ---------------------------------------- tests/test_wcs.py | 67 --------------------- 6 files changed, 2 insertions(+), 283 deletions(-) delete mode 100644 src/bcdata/wcs.py delete mode 100644 tests/test_wcs.py diff --git a/README.md b/README.md index 4c1c073..a337ec2 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,13 @@ [![Actions Status](https://github.com/smnorris/bcdata/actions/workflows/tests.yml/badge.svg)](https://github.com/smnorris/bcdata/actions?query=workflow%3Atests) [![pypi](https://img.shields.io/pypi/v/bcdata.svg)](https://pypi.python.org/pypi/bcdata/) -Python and command line tools for quick access to DataBC geo-data available via WFS/WCS. +Python and command line tools for quick access to DataBC geo-data available via WFS. There is a [wealth of British Columbia geographic information available as open data](https://catalogue.data.gov.bc.ca/dataset?download_audience=Public), but most sources are available only via WFS - and the syntax to download WFS data via `ogr2ogr` and/or `curl/wget` can be awkward. -This tool attempts to simplify downloads of BC geographic data and smoothly integrate with PostGIS and Python GIS tools like `geopandas`, `fiona` and `rasterio`. The tool only accesses geographic data [avialable via WFS/WCS](https://bcgov.github.io/data-publication/pages/tips_tricks_webservices.html) - for other BC open data, download the files directly with `requests` / `curl` / `ogr2ogr` / etc (or the [bcdata R package](https://github.com/bcgov/bcdata)). +This tool attempts to simplify downloads of BC geographic data and smoothly integrate with PostGIS and Python GIS tools like `geopandas`, `fiona` and `rasterio`. The tool only accesses geographic data [avialable via WFS](https://bcgov.github.io/data-publication/pages/tips_tricks_webservices.html) - for other BC open data, download the files directly with `requests` / `curl` / `ogr2ogr` / etc (or the [bcdata R package](https://github.com/bcgov/bcdata)). If downloads are failing, consult the [BCGov Data Systems and Services uptime monitor](https://uptime.com/statuspage/bcgov-dss) - this tool depends on BC Open Geospatial Web Services API at [openmaps.gov.bc.ca](https://openmaps.gov.bc.ca/geo/pub/ows?service=WFS&request=Getcapabilities) and the [BC Data Catalogue API](https://catalogue.data.gov.bc.ca/dataset/bc-data-catalogue-api). @@ -98,7 +98,6 @@ Options: Commands: bc2pg Load a DataBC WFS layer to a postgres db cat Write DataBC features to stdout as GeoJSON feature objects. - dem Dump BC DEM to TIFF dump Write DataBC features to stdout as GeoJSON feature collection. info Print basic metadata about a DataBC WFS layer as JSON. list List DataBC layers available via WFS @@ -166,29 +165,6 @@ Options: --help Show this message and exit. ``` -#### dem - -``` -$ bcdata dem --help - -Usage: bcdata dem [OPTIONS] - - Dump BC DEM to TIFF - -Options: - -o, --out_file TEXT Output file - --bounds TEXT Bounds: "left bottom right top" or "[left, - bottom, right, top]". Coordinates are BC - Albers (default) or --bounds_crs [required] - --bounds-crs TEXT CRS of provided bounds - -r, --resolution INTEGER - -a, --align Align provided bounds to provincial standard - -i, --interpolation [nearest|bilinear|bicubic] - -v, --verbose Increase verbosity. - -q, --quiet Decrease verbosity. - --help Show this message and exit. -``` - #### dump ``` diff --git a/scripts/document_cli.sh b/scripts/document_cli.sh index ce69928..220f8d8 100755 --- a/scripts/document_cli.sh +++ b/scripts/document_cli.sh @@ -20,12 +20,6 @@ printf '```\n$ bcdata cat --help\n\n' >> cli.md bcdata cat --help >> cli.md printf '```\n' >> cli.md -printf '\n' >> cli.md -printf '#### dem \n\n' >> cli.md -printf '```\n$ bcdata dem --help\n\n' >> cli.md -bcdata dem --help >> cli.md -printf '```\n' >> cli.md - printf '\n' >> cli.md printf '#### dump \n\n' >> cli.md printf '```\n$ bcdata dump --help\n\n' >> cli.md diff --git a/src/bcdata/__init__.py b/src/bcdata/__init__.py index e5cbd64..ed66091 100644 --- a/src/bcdata/__init__.py +++ b/src/bcdata/__init__.py @@ -3,7 +3,6 @@ from .bc2pg import bc2pg as bc2pg from .bcdc import get_table_definition as get_table_definition from .bcdc import get_table_name as get_table_name -from .wcs import get_dem as get_dem from .wfs import get_count as get_count from .wfs import get_data as get_data from .wfs import get_sortkey as get_sortkey diff --git a/src/bcdata/cli.py b/src/bcdata/cli.py index f0b213b..d7528c9 100644 --- a/src/bcdata/cli.py +++ b/src/bcdata/cli.py @@ -65,14 +65,6 @@ def bounds_handler(ctx, param, value): help='Bounds: "left bottom right top" or "[left, bottom, right, top]". Coordinates are BC Albers (default) or --bounds_crs', ) -bounds_opt_dem = click.option( - "--bounds", - required=True, - default=None, - callback=bounds_handler, - help='Bounds: "left bottom right top" or "[left, bottom, right, top]". Coordinates are BC Albers (default) or --bounds_crs', -) - lowercase_opt = click.option( "--lowercase", "-l", is_flag=True, help="Write column/properties names as lowercase" @@ -137,51 +129,6 @@ def info(dataset, indent, meta_member, verbose, quiet): click.echo(json.dumps(info, indent=indent)) -@cli.command() -@click.option("--out_file", "-o", help="Output file", default="dem25.tif") -@bounds_opt_dem -@click.option( - "--bounds-crs", - help="CRS of provided bounds", - default="EPSG:3005", -) -@click.option("--resolution", "-r", type=int, default=25) -@click.option( - "--align", - "-a", - is_flag=True, - help="Align provided bounds to provincial standard", -) -@click.option( - "--interpolation", - "-i", - type=click.Choice(["nearest", "bilinear", "bicubic"], case_sensitive=False), -) -@verbose_opt -@quiet_opt -def dem( - bounds, - bounds_crs, - align, - out_file, - resolution, - interpolation, - verbose, - quiet, -): - """Dump BC DEM to TIFF""" - verbosity = verbose - quiet - configure_logging(verbosity) - bcdata.get_dem( - bounds, - out_file=out_file, - align=align, - src_crs=bounds_crs, - resolution=resolution, - interpolation=interpolation, - ) - - @cli.command() @click.argument("dataset", type=click.STRING) @click.option( diff --git a/src/bcdata/wcs.py b/src/bcdata/wcs.py deleted file mode 100644 index 03a10c7..0000000 --- a/src/bcdata/wcs.py +++ /dev/null @@ -1,130 +0,0 @@ -import logging -from math import trunc - -import rasterio -import requests -import stamina - -log = logging.getLogger(__name__) - -WCS_URL = "https://openmaps.gov.bc.ca/om/wcs" - - -class ServiceException(Exception): - pass - - -def align_bounds(bounds): - """ - Adjust input bounds to align with Hectares BC raster - (round bounds to nearest 100m, then shift by 12.5m) - """ - ll = [((trunc(b / 100) * 100) - 12.5) for b in bounds[:2]] - ur = [(((trunc(b / 100) + 1) * 100) + 87.5) for b in bounds[2:]] - return (ll[0], ll[1], ur[0], ur[1]) - - -@stamina.retry(on=requests.HTTPError, timeout=60) -def make_request(payload): - r = requests.get( - WCS_URL, - params=payload, - headers={"User-Agent": "bcdata.py ({bcdata.__version__})"}, - ) - log.debug(r.url) - if r.status_code == 200: - return r - elif r.status_code in [400, 401, 404]: - log.error(f"HTTP error {r.status_code}") - log.error(f"Response headers: {r.headers}") - log.error(f"Response text: {r.text}") - raise ServiceException(r.text) # presumed request error - elif r.status_code in [500, 502, 503, 504]: # presumed serivce error, retry - log.warning(f"HTTP error: {r.status_code}, retrying") - log.warning(f"Response headers: {r.headers}") - log.warning(f"Response text: {r.text}") - r.raise_for_status() - - -def get_dem( - bounds, - out_file="dem.tif", - src_crs="EPSG:3005", - dst_crs="EPSG:3005", - resolution=25, - align=False, - interpolation=None, - as_rasterio=False, -): - """Get TRIM DEM for provided bounds, write to GeoTIFF.""" - # align bounds if specified (and bounds are BC Albers CRS) - if align: - if src_crs.upper() == "EPSG:3005" and dst_crs.upper() == "EPSG:3005": - bounds = align_bounds(bounds) - else: - raise ValueError( - f"Target CRS is {dst_crs}, align is only valid for BC Albers based bounds and outputs" - ) - - bbox = ",".join([str(b) for b in bounds]) - - # do not upsample - if resolution < 25: - raise ValueError("Resolution requested must be 25m or greater") - - # if downsampling, default to bilinear (the server defaults to nearest) - if resolution > 25 and not interpolation: - log.info("Interpolation not specified, defaulting to bilinear") - interpolation = "bilinear" - - # if specifying interpolation method, there has to actually be a - # resampling requested - resolution can't be the native 25m - if interpolation and resolution == 25: - raise ValueError( - "Requested coverage at native resolution, no resampling required, interpolation {} invalid" - ) - - # make sure interpolation is valid - if interpolation: - valid_interpolations = ["nearest", "bilinear", "bicubic"] - if interpolation not in valid_interpolations: - raise ValueError( - "Interpolation {} invalid. Valid keys are: {}".format( - interpolation, ",".join(valid_interpolations) - ) - ) - - # build request - payload = { - "service": "WCS", - "version": "1.0.0", - "request": "GetCoverage", - "coverage": "pub:bc_elevation_25m_bcalb", - "Format": "GeoTIFF", - "bbox": bbox, - "CRS": src_crs, - "RESPONSE_CRS": dst_crs, - "resx": str(resolution), - "resy": str(resolution), - } - if interpolation: - payload["INTERPOLATION"] = interpolation - - # request data from WCS - log.debug(payload) - r = make_request(payload) - if r.headers["Content-Type"] == "image/tiff": - with open(out_file, "wb") as file: - file.write(r.content) - elif r.headers["Content-Type"] == "application/vnd.ogc.se_xml;charset=UTF-8": - raise RuntimeError( - "WCS request {} failed with error {}".format(r.url, str(r.content.decode("utf-8"))) - ) - else: - raise RuntimeError( - "WCS request {} failed, content type {}".format(r.url, str(r.headers["Content-Type"])) - ) - if as_rasterio: - return rasterio.open(out_file, "r") - else: - return out_file diff --git a/tests/test_wcs.py b/tests/test_wcs.py deleted file mode 100644 index 2a9a921..0000000 --- a/tests/test_wcs.py +++ /dev/null @@ -1,67 +0,0 @@ -import os - -import pytest -import rasterio -from rasterio.coords import BoundingBox - -import bcdata - - -def test_dem(tmpdir): - bounds = [1046891, 704778, 1055345, 709629] - out_file = bcdata.get_dem(bounds, os.path.join(tmpdir, "test_dem.tif")) - assert os.path.exists(out_file) - with rasterio.open(out_file) as src: - stats = [ - { - "min": float(b.min()), - "max": float(b.max()), - "mean": float(b.mean()), - } - for b in src.read() - ] - assert stats[0]["max"] == 3982 - - -def test_dem_align(tmpdir): - bounds = [1046891, 704778, 1055345, 709629] - out_file = bcdata.get_dem(bounds, os.path.join(tmpdir, "test_dem_align.tif"), align=True) - assert os.path.exists(out_file) - with rasterio.open(out_file) as src: - bounds = src.bounds - bbox = BoundingBox(1046787.5, 704687.5, 1055487.5, 709787.5) - assert bounds == bbox - - -def test_dem_rasterio(tmpdir): - bounds = [1046891, 704778, 1055345, 709629] - src = bcdata.get_dem(bounds, os.path.join(tmpdir, "test_dem_rasterio.tif"), as_rasterio=True) - stats = [ - {"min": float(b.min()), "max": float(b.max()), "mean": float(b.mean())} for b in src.read() - ] - assert stats[0]["max"] == 3982 - - -# interpolation takes a while to run, comment out for for faster tests -# def test_dem_resample(tmpdir): -# bounds = [1046891, 704778, 1055345, 709629] -# out_file = bcdata.get_dem(bounds, os.path.join(tmpdir, "test_dem.tif"), interpolation="bilinear", resolution=50) -# assert os.path.exists(out_file) -# with rasterio.open(out_file) as src: -# stats = [{'min': float(b.min()), -# 'max': float(b.max()), -# 'mean': float(b.mean()) -# } for b in src.read()] -# assert stats[0]['max'] == 3956.0 - - -def test_dem_invalid_resample1(): - with pytest.raises(ValueError): - bounds = [1046891, 704778, 1055345, 709629] - bcdata.get_dem(bounds, "test_dem.tif", interpolation="cubic", resolution=50) - - -def test_dem_invalid_resample2(): - with pytest.raises(ValueError): - bounds = [1046891, 704778, 1055345, 709629] - bcdata.get_dem(bounds, "test_dem.tif", interpolation="bilinear") From 43d5047ff971dd2d9f4fce744c0944831cbc9a70 Mon Sep 17 00:00:00 2001 From: Simon Norris Date: Wed, 16 Jul 2025 14:46:05 -0700 Subject: [PATCH 2/2] fix #217 by using new dataset name, unsure if test is still required for new layer but it passes if comments are not queried --- tests/test_bcdc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_bcdc.py b/tests/test_bcdc.py index 4fb3c31..6a15f88 100644 --- a/tests/test_bcdc.py +++ b/tests/test_bcdc.py @@ -57,10 +57,10 @@ def test_get_table_definition_format_multi_nopreview(): def test_get_table_definition_format_multi_nolayer(): table_definition = bcdc.get_table_definition( - "WHSE_HUMAN_CULTURAL_ECONOMIC.HIST_HISTORIC_ENVIRONMENTS_SP" + "WHSE_HUMAN_CULTURAL_ECONOMIC.HIST_HISTORIC_ENVIRONMNT_PA_SV" ) assert table_definition["description"] - assert table_definition["comments"] + # assert table_definition["comments"] there are no comments associated with this dataset assert table_definition["schema"]