From f5bf08b5852b875ea77ac9c7d845730f1dfa5ef6 Mon Sep 17 00:00:00 2001 From: Jack-Hayes Date: Tue, 14 Oct 2025 14:41:17 +0200 Subject: [PATCH 1/2] speed up binder build and remove redundant files --- .condarc | 4 +++ .github/workflows/binder-cache.yml | 21 ---------------- .github/workflows/binder-warmup.yml | 21 ++++++++++++++++ README.md | 31 ++++++++++++++--------- binder/environment.yml | 15 +++++++----- binder/postBuild | 10 -------- dev.pixi.toml | 38 ----------------------------- 7 files changed, 54 insertions(+), 86 deletions(-) create mode 100644 .condarc delete mode 100644 .github/workflows/binder-cache.yml create mode 100644 .github/workflows/binder-warmup.yml delete mode 100644 binder/postBuild delete mode 100644 dev.pixi.toml diff --git a/.condarc b/.condarc new file mode 100644 index 0000000..64a367d --- /dev/null +++ b/.condarc @@ -0,0 +1,4 @@ +channels: + - conda-forge +channel_priority: strict +show_channel_urls: true diff --git a/.github/workflows/binder-cache.yml b/.github/workflows/binder-cache.yml deleted file mode 100644 index a318543..0000000 --- a/.github/workflows/binder-cache.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Cache Binder Image - -on: - push: - branches: - - main - -jobs: - build_binder: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Pre-warm Binder cache on mybinder.org - uses: machine-learning-apps/repo2docker-action@master - with: - NO_PUSH: true - # trigger a build on mybinder.org - # uses the Git reference (e.g., 'refs/heads/main') as the image tag. - MYBINDERORG_TAG: ${{ github.ref }} \ No newline at end of file diff --git a/.github/workflows/binder-warmup.yml b/.github/workflows/binder-warmup.yml new file mode 100644 index 0000000..0373578 --- /dev/null +++ b/.github/workflows/binder-warmup.yml @@ -0,0 +1,21 @@ +name: Binder cache warmup + +on: + push: + branches: [ main ] + schedule: + - cron: "0 3 * * 0" # weekly, Sunday 03:00 UTC + workflow_dispatch: + +jobs: + warmup: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Trigger mybinder.org build + run: | + set -eux + REF="${{ github.ref_name }}" + if [ -z "$REF" ] || [ "$REF" = "refs/heads/" ]; then REF="main"; fi + curl -LsS -o /dev/null -w "%{http_code}\n" \ + "https://mybinder.org/build/gh/${{ github.repository }}/$REF" || true diff --git a/README.md b/README.md index 3a9d3b5..6de540b 100644 --- a/README.md +++ b/README.md @@ -16,16 +16,18 @@ Many thanks to banesullivan's [localtileserver](https://github.com/banesullivan/ Launch on Binder -What happens when you click the button (suggested to right click -> open in new tab): -- A temporary, free session is launched on mybinder.org (a community, shared service) that builds a Docker image from this repo’s environment files. -- First launch after a change can take several minutes while the image builds. Subsequent launches are faster (cached), but caches may be evicted at any time. -- Sessions are ephemeral (typically 1–2 hours) and resources are limited. Your session will shut down if idle or when it times out. +What happens when you click the button: +- mybinder.org uses repo2docker to build a Docker image for this repo using binder/environment.yml and runtime.txt (Python 3.11 base) +- The conda environment is solved/installed with mamba from conda-forge; during the build, pip installs this repo in editable mode (-e .) +- The built image is cached by mybinder.org for the given Git ref; first builds can take several minutes, later builds are usually faster until the cache is evicted +- Sessions are ephemeral and resource-limited; they time out when idle (~10 mins) or after a short lifetime ## Notes and expectations - First launch: building can take several minutes; please be patient. -- If you see “Building” or “Pending,” Binder is queuing your session or compiling the environment. +- If you see "Building' or "Pending," Binder is queuing your session or compiling the environment. - If tiles don’t appear immediately after "Fetch & Display Data," wait a few seconds. +- Don't sample large areas -- this will drop performace. Less than 30km2 is a good bet. - The app depends on external open data services; occasional slowdowns or gaps can occur. ## What you can do @@ -33,17 +35,24 @@ What happens when you click the button (suggested to right click -> open in new - Pick a date range, draw an Area of Interest (blue) on the map, then click "Fetch & Display Data." - Toggle layers in the layer control: - RGB composite (Sentinel‑2) [lowest cloud coverage in date range is selected] - - NDVI and a basic built-up index [lowest cloud coverage in date range is selected] - SAR backscatter (Sentinel‑1) [acquisition closest to Sentinel-2 image is selected] + - Coopernicus 30-GLO DEM Multidirectional Hillshade - ESA WorldCover (classified land cover) [2021] - Draw a red Region of Interest (ROI) to sample statistics from visible layers. ## How it works (high level) -- You draw an area; the app searches a STAC catalog (Microsoft Planetary Computer) for relevant items. -- It selects a recent, low‑cloud Sentinel‑2 scene and optionally a time‑aligned Sentinel‑1 scene. -- On the fly, it computes quicklook products (RGB, NDVI, built‑up index), and prepares tiles from in‑memory COGs. -- Tiles are served inside the session and displayed via ipyleaflet (no data leaves your browser other than tile requests to the Binder session) +- You draw an area; the app searches the Microsoft Planetary Computer STAC API for relevant items +- It selects a low‑cloud Sentinel‑2 scene and a time‑aligned Sentinel‑1 scene +- It computes quicklook products: + - Sentinel‑2 RGB + - Sentinel‑1 VV/VH in decibels + - Multidirectional hillshade from Copernicus GLO‑30 + - ESA WorldCover mapped to RGB colors +- Rasters are written to in‑memory COGs and served by localtileserver running in your Binder session. +- ipyleaflet requests tiles via Jupyter Server Proxy (e.g., /proxy/{port}); the browser only talks to the Binder pod, while the pod fetches data from the Planetary Computer. + +> **_NOTE:_** All layers / STAC endpoints are oprtional ## Contributing @@ -56,4 +65,4 @@ What happens when you click the button (suggested to right click -> open in new - Conda/mamba: create an environment similar to binder/environment.yml, then `pip install -e .` - Or use your own stack and ensure GDAL/Rasterio/PROJ are consistent. - Run notebooks locally with JupyterLab or launch the Voila app: - - `voila notebooks/app.ipynb \ No newline at end of file + - `voila notebooks/app.ipynb` \ No newline at end of file diff --git a/binder/environment.yml b/binder/environment.yml index d59b52e..43691d2 100644 --- a/binder/environment.yml +++ b/binder/environment.yml @@ -8,16 +8,19 @@ dependencies: - voila - ipywidgets - ipyleaflet - - numpy + - numpy=2.3.* - pandas - - xarray - - geopandas - - rioxarray - - rasterio - - shapely + - xarray=2025.10.* + - geopandas=1.1.* + - rioxarray=0.19.* + - rasterio=1.4.* + - shapely=2.1.* + - gdal=3.10.* - matplotlib - odc-stac - pystac-client - planetary-computer - localtileserver - jupyter-server-proxy + - pip: + - -e . diff --git a/binder/postBuild b/binder/postBuild deleted file mode 100644 index 52631e2..0000000 --- a/binder/postBuild +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -# binder/postBuild -# runs after the environment is built -set -euxo pipefail - -# Install the src/planetquery package in editable mode -pip install -e . - -# Explicitly enable the proxy extension (conda will do the but we want to be certain) -jupyter server extension enable jupyter_server_proxy || true \ No newline at end of file diff --git a/dev.pixi.toml b/dev.pixi.toml deleted file mode 100644 index 9002539..0000000 --- a/dev.pixi.toml +++ /dev/null @@ -1,38 +0,0 @@ -[project] -name = "planetquery" -description = "PlanetQuery demo using STAC + ipyleaflet" - -[env] -python = "3.11.12" - -[deps.conda] -channels = ["conda-forge"] -packages = [ - "python=3.11.12", - "ipykernel", - "jupyterlab", - "voila", - "notebook", - "ipywidgets", - "ipyleaflet", - "matplotlib", - "numpy", - "pandas", - "xarray", - "geopandas", - "rioxarray", - "rasterio", - "shapely", - "gdal", - "nodejs" -] - -[deps.pip] -packages = [ - "pystac-client", - "planetary-computer", - "odc-stac", - "localtileserver", - "localtileserver-client", - "matplotlib-inline" -] From befda1c1acf94a339ce6bf161c893531b71dac5d Mon Sep 17 00:00:00 2001 From: Jack-Hayes Date: Tue, 14 Oct 2025 14:50:39 +0200 Subject: [PATCH 2/2] env yml pip points one level up to root --- binder/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/binder/environment.yml b/binder/environment.yml index 43691d2..4acafc3 100644 --- a/binder/environment.yml +++ b/binder/environment.yml @@ -23,4 +23,4 @@ dependencies: - localtileserver - jupyter-server-proxy - pip: - - -e . + - -e ..