Feature/documentation

.gitignore

@@ -144,6 +144,7 @@ data.zip
 data/*
 data1
 data
+data_raw/
 
 data_mid/
 data_out/

.pre-commit-config.yaml

@@ -1,4 +1,13 @@
 fail_fast: true
+
+exclude: |
+  (?x)^(
+      notebooks/process_sandbox\.ipynb|
+      src/cerf/data_acquisition_scrape\.py|
+      src/disaster_charter/data_acquisition_scrape\.py|
+      src/glide/data_acquisition_scrape\.py
+  )$
+
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.6.8

Makefile

@@ -24,6 +24,30 @@ clean:
 	@rm -rf .venv
 	@poetry env remove --all
 
+run_gdacs_download:
+	@echo "Running GDACS download"
+	@poetry run python -m src.gdacs.data_acquisition_api
+
+run_glide_download:
+	@echo "Running Glide download"
+	@poetry run python -m src.glide.data_acquisition_scrape
+
+run_cerf_download:
+	@echo "Running CERF download"
+	@poetry run python -m src.cerf.data_acquisition_scrape
+
+run_disaster_charter_download:
+	@echo "Running Disaster-Charter download"
+	@poetry run python -m src.disaster_charter.data_acquisition_scrape
+
+run_idus_download:
+	@echo "Downloading IDUS dump → data_raw/idmc_idu/idus_all.json"
+	@mkdir -p data_raw/idmc_idu
+	@curl -L --compressed \
+		-o data_raw/idmc_idu/idus_all.json \
+		"https://helix-copilot-prod-helix-media-external.s3.amazonaws.com/external-media/api-dump/idus-all/2025-06-04-10-00-32/5mndO/idus_all.json"
+	@echo "✅  Saved (decompressed): data_raw/idmc_idu/idus_all.json"
+
 run_glide_normal:
 	@echo "Running Glide normalisation"
 	@poetry run python -m src.glide.data_normalisation_glide

docs/DATASETS.md

@@ -21,6 +21,21 @@ update any of the datasets:
 **Important**: Always preserve the folder structure to avoid breaking downstream
 processes.
 
+## API-Based Dataset: GDACS
+
+The GDACS dataset is unique in that it provides an official API, made available directly by the GDACS team.
+
+To update this dataset:
+
+1. Modify the date range in the `src/gdacs/data_acquisition_api.py` script and the `main` method.
+2. Run the following command:
+
+    ```sh
+    make run_gdacs_download
+    ```
+
+This process will automatically download and save the updated records to the appropriate location.
+
 ## Web-Scraped Legacy Datasets (Now Blocked or Fragile)
 
 Some datasets were initially extracted using automated **web scraping scripts**.
@@ -31,7 +46,6 @@ category:
 - CERF Activations
 - Disaster Charter Activations
 - GLIDE Events
-- GDACS Events
 - WFP ADAM
 
 ### How to Update These