From 5301dbe12f153dbbd9f83ba08c90a1145c1f03bc Mon Sep 17 00:00:00 2001 From: Marco Otte-Witte Date: Wed, 2 Apr 2025 14:49:31 +0200 Subject: [PATCH 1/5] remove info_xy columns --- cli/src/bin/db.rs | 116 +-------------------- db/schema.sql | 20 +--- db/src/entities/stations.rs | 174 ++------------------------------ db/src/test_helpers/stations.rs | 38 +------ web/tests/api/places_test.rs | 3 +- 5 files changed, 12 insertions(+), 339 deletions(-) diff --git a/cli/src/bin/db.rs b/cli/src/bin/db.rs index 31d3284..4769d5f 100644 --- a/cli/src/bin/db.rs +++ b/cli/src/bin/db.rs @@ -174,24 +174,6 @@ struct StationRecord { pub latitude: Option, pub longitude: Option, pub country: Option, - pub info_de: Option, - pub info_en: Option, - pub info_es: Option, - pub info_fr: Option, - pub info_it: Option, - pub info_nb: Option, - pub info_nl: Option, - pub info_cs: Option, - pub info_da: Option, - pub info_hu: Option, - pub info_ja: Option, - pub info_ko: Option, - pub info_pl: Option, - pub info_pt: Option, - pub info_ru: Option, - pub info_sv: Option, - pub info_tr: Option, - pub info_zh: Option, } async fn sync(config: &Config) -> Result { @@ -225,28 +207,10 @@ async fn sync(config: &Config) -> Result { uic, latitude, longitude, - country, - info_de, - info_en, - info_es, - info_fr, - info_it, - info_nb, - info_nl, - info_cs, - info_da, - info_hu, - info_ja, - info_ko, - info_pl, - info_pt, - info_ru, - info_sv, - info_tr, - info_zh + country ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ? ) "#, ) @@ -256,24 +220,6 @@ async fn sync(config: &Config) -> Result { .bind(station.latitude) .bind(station.longitude) .bind(station.country) - .bind(station.info_de) - .bind(station.info_en) - .bind(station.info_es) - .bind(station.info_fr) - .bind(station.info_it) - .bind(station.info_nb) - .bind(station.info_nl) - .bind(station.info_cs) - .bind(station.info_da) - .bind(station.info_hu) - .bind(station.info_ja) - .bind(station.info_ko) - .bind(station.info_pl) - .bind(station.info_pt) - .bind(station.info_ru) - .bind(station.info_sv) - .bind(station.info_tr) - .bind(station.info_zh) .execute(&mut conn) .await?; i += 1; @@ -306,29 +252,9 @@ fn prepare_station(record: StringRecord, i: i32) -> Result Result { let id = id .parse::() @@ -383,24 +291,6 @@ fn prepare_station(record: StringRecord, i: i32) -> Result Err(anyhow!("Invalid data in line {}!", i)), diff --git a/db/schema.sql b/db/schema.sql index ac78ad6..4b465ef 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -4,25 +4,7 @@ CREATE TABLE stations ( uic TEXT NOT NULL, latitude REAL, longitude REAL, - country TEXT, - info_de TEXT, - info_en TEXT, - info_es TEXT, - info_fr TEXT, - info_it TEXT, - info_nb TEXT, - info_nl TEXT, - info_cs TEXT, - info_da TEXT, - info_hu TEXT, - info_ja TEXT, - info_ko TEXT, - info_pl TEXT, - info_pt TEXT, - info_ru TEXT, - info_sv TEXT, - info_tr TEXT, - info_zh TEXT + country TEXT ); CREATE UNIQUE INDEX stations_id_idx ON stations (id); diff --git a/db/src/entities/stations.rs b/db/src/entities/stations.rs index 6c83985..4c426be 100644 --- a/db/src/entities/stations.rs +++ b/db/src/entities/stations.rs @@ -20,24 +20,6 @@ pub struct Station { pub latitude: Option, pub longitude: Option, pub country: Option, - pub info_de: Option, - pub info_en: Option, - pub info_es: Option, - pub info_fr: Option, - pub info_it: Option, - pub info_nb: Option, - pub info_nl: Option, - pub info_cs: Option, - pub info_da: Option, - pub info_hu: Option, - pub info_ja: Option, - pub info_ko: Option, - pub info_pl: Option, - pub info_pt: Option, - pub info_ru: Option, - pub info_sv: Option, - pub info_tr: Option, - pub info_zh: Option, } #[derive(Deserialize, Validate, Clone)] @@ -59,24 +41,6 @@ pub struct StationChangeset { pub longitude: Option, #[cfg_attr(feature = "test-helpers", dummy(faker = "CountryName()"))] pub country: Option, - pub info_de: Option, - pub info_en: Option, - pub info_es: Option, - pub info_fr: Option, - pub info_it: Option, - pub info_nb: Option, - pub info_nl: Option, - pub info_cs: Option, - pub info_da: Option, - pub info_hu: Option, - pub info_ja: Option, - pub info_ko: Option, - pub info_pl: Option, - pub info_pt: Option, - pub info_ru: Option, - pub info_sv: Option, - pub info_tr: Option, - pub info_zh: Option, } pub async fn load_all( @@ -90,25 +54,7 @@ pub async fn load_all( uic, latitude, longitude, - country, - info_de, - info_en, - info_es, - info_fr, - info_it, - info_nb, - info_nl, - info_cs, - info_da, - info_hu, - info_ja, - info_ko, - info_pl, - info_pt, - info_ru, - info_sv, - info_tr, - info_zh + country FROM stations" ) @@ -129,25 +75,7 @@ pub async fn load_all_within_limit( uic, latitude, longitude, - country, - info_de, - info_en, - info_es, - info_fr, - info_it, - info_nb, - info_nl, - info_cs, - info_da, - info_hu, - info_ja, - info_ko, - info_pl, - info_pt, - info_ru, - info_sv, - info_tr, - info_zh + country FROM stations LIMIT @@ -171,25 +99,7 @@ pub async fn load( uic, latitude, longitude, - country, - info_de, - info_en, - info_es, - info_fr, - info_it, - info_nb, - info_nl, - info_cs, - info_da, - info_hu, - info_ja, - info_ko, - info_pl, - info_pt, - info_ru, - info_sv, - info_tr, - info_zh + country FROM stations WHERE @@ -219,49 +129,13 @@ pub async fn search_by_name( uic, latitude, longitude, - country, - info_de, - info_en, - info_es, - info_fr, - info_it, - info_nb, - info_nl, - info_cs, - info_da, - info_hu, - info_ja, - info_ko, - info_pl, - info_pt, - info_ru, - info_sv, - info_tr, - info_zh + country FROM stations WHERE uic IS NOT NULL AND ( name LIKE $1 - OR info_de LIKE $1 - OR info_en LIKE $1 - OR info_es LIKE $1 - OR info_fr LIKE $1 - OR info_it LIKE $1 - OR info_nb LIKE $1 - OR info_nl LIKE $1 - OR info_cs LIKE $1 - OR info_da LIKE $1 - OR info_hu LIKE $1 - OR info_ja LIKE $1 - OR info_ko LIKE $1 - OR info_pl LIKE $1 - OR info_pt LIKE $1 - OR info_ru LIKE $1 - OR info_sv LIKE $1 - OR info_tr LIKE $1 - OR info_zh LIKE $1 ) ORDER BY name @@ -298,25 +172,7 @@ pub async fn search_by_position( uic, latitude, longitude, - country, - info_de, - info_en, - info_es, - info_fr, - info_it, - info_nb, - info_nl, - info_cs, - info_da, - info_hu, - info_ja, - info_ko, - info_pl, - info_pt, - info_ru, - info_sv, - info_tr, - info_zh + country FROM stations WHERE @@ -367,25 +223,7 @@ pub async fn search_by_name_and_position( uic, latitude, longitude, - country, - info_de, - info_en, - info_es, - info_fr, - info_it, - info_nb, - info_nl, - info_cs, - info_da, - info_hu, - info_ja, - info_ko, - info_pl, - info_pt, - info_ru, - info_sv, - info_tr, - info_zh + country FROM stations WHERE diff --git a/db/src/test_helpers/stations.rs b/db/src/test_helpers/stations.rs index 2204769..3652585 100644 --- a/db/src/test_helpers/stations.rs +++ b/db/src/test_helpers/stations.rs @@ -8,31 +8,13 @@ pub async fn create(station: StationChangeset, db: &DbPool) -> Result Result Date: Wed, 2 Apr 2025 14:59:21 +0200 Subject: [PATCH 2/5] prepare nominatim import --- cli/src/bin/db.rs | 3 +++ docker-compose.yml | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 docker-compose.yml diff --git a/cli/src/bin/db.rs b/cli/src/bin/db.rs index 4769d5f..bd128d0 100644 --- a/cli/src/bin/db.rs +++ b/cli/src/bin/db.rs @@ -197,6 +197,9 @@ async fn sync(config: &Config) -> Result { let record = record.context("Failed to read record from CSV file!")?; let station = prepare_station(record, i)?; + // TODO: get name (all languages), country name (all languages), country code, postcode, city (all languages), display name (all languages), street (all languages) + // localhost:8080/reverse?format=jsonv2&lat=48.876742&lon=2.358424&addressdetails=1&namedetails=1 + sqlx::query( r#" INSERT INTO diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..3f68e7c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,17 @@ +services: + nominatim: + image: "mediagis/nominatim:5.1" + restart: always + environment: + PBF_URL: "https://download.geofabrik.de/europe/monaco-latest.osm.pbf" + REPLICATION_URL: "https://download.geofabrik.de/europe-updates/" + REVERSE_ONLY: true + IMPORT_GB_POSTCODES: true + volumes: + - db:/var/lib/postgresql/16/main + - flatnode:/nominatim/flatnode + ports: + - 8080:8080 +volumes: + db: + flatnode: From 6ad989dad672baeee52bcb473dd2174ccdef6de6 Mon Sep 17 00:00:00 2001 From: Marco Otte-Witte Date: Wed, 2 Apr 2025 15:53:30 +0200 Subject: [PATCH 3/5] add JSON data column --- db/schema.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db/schema.sql b/db/schema.sql index 4b465ef..7154193 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -4,7 +4,8 @@ CREATE TABLE stations ( uic TEXT NOT NULL, latitude REAL, longitude REAL, - country TEXT + country TEXT, + data JSON ); CREATE UNIQUE INDEX stations_id_idx ON stations (id); From 23c1123fabc69a34c1edda34ee85121d8883c82a Mon Sep 17 00:00:00 2001 From: Marco Otte-Witte Date: Wed, 2 Apr 2025 15:53:41 +0200 Subject: [PATCH 4/5] load all of Europe into Nominatim --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3f68e7c..f2954ca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ services: image: "mediagis/nominatim:5.1" restart: always environment: - PBF_URL: "https://download.geofabrik.de/europe/monaco-latest.osm.pbf" + PBF_URL: "https://download.geofabrik.de/europe-latest.osm.pbf" REPLICATION_URL: "https://download.geofabrik.de/europe-updates/" REVERSE_ONLY: true IMPORT_GB_POSTCODES: true From 7b7d181eec9d87bee39ae06fe73e78567e7e53ce Mon Sep 17 00:00:00 2001 From: Marco Otte-Witte Date: Wed, 2 Apr 2025 16:36:18 +0200 Subject: [PATCH 5/5] get metadata per station --- Cargo.lock | 1 + cli/Cargo.toml | 3 +- cli/src/bin/db.rs | 80 +++++++++++++++++++++++++++++++++++++---------- 3 files changed, 67 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a001175..861af49 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2317,6 +2317,7 @@ dependencies = [ "liquid", "reqwest", "restations-config", + "serde_json", "sqlx", "tempfile", "tokio", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 232a3a8..2c275f0 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -25,7 +25,8 @@ guppy = "0.17" include_dir = "0.7" liquid = "~0.26" restations-config = { path = "../config" } -reqwest = { version = "0.12", features = ["stream"] } +reqwest = { version = "0.12", features = ["stream", "json"] } +serde_json = "1.0" sqlx = { version = "0.8", features = [ "runtime-tokio", "tls-rustls", diff --git a/cli/src/bin/db.rs b/cli/src/bin/db.rs index bd128d0..0b6eb9a 100644 --- a/cli/src/bin/db.rs +++ b/cli/src/bin/db.rs @@ -3,12 +3,13 @@ use clap::{Parser, Subcommand}; use csv_async::{AsyncReaderBuilder, StringRecord, Trim}; use futures::stream::TryStreamExt; use guppy::{Version, VersionReq}; -use reqwest::Client; +use reqwest::{header::USER_AGENT, Client}; use restations_cli::util::ui::UI; use restations_config::DatabaseConfig; use restations_config::{load_config, parse_env, Config, Environment}; +use serde_json::Value; use sqlx::sqlite::{SqliteConnectOptions, SqliteConnection}; -use sqlx::{ConnectOptions, Connection}; +use sqlx::{ConnectOptions, Connection, Row}; use std::path::PathBuf; use std::process::{ExitCode, Stdio}; use tokio::{ @@ -60,6 +61,8 @@ enum Commands { Create, #[command(about = "Synchronize the database with the source data")] Sync, + #[command(about = "Augment the data with additional location data")] + Augment, #[command(about = "Generate query metadata to support offline compile-time verification")] Prepare, } @@ -97,6 +100,20 @@ async fn cli(ui: &mut UI<'_>, cli: Cli) -> Result<(), anyhow::Error> { ui.success(&format!("{} stations synchronized.", stations)); Ok(()) } + Commands::Augment => { + ui.info(&format!("Augmenting {} database…", &cli.env)); + ui.indent(); + let result = augment(&config) + .await + .context("Could not augment database!"); + ui.outdent(); + let (augmented_stations, unavailable_stations) = result?; + ui.success(&format!( + "{} stations augmented, no data found for {} stations.", + augmented_stations, unavailable_stations + )); + Ok(()) + } Commands::Prepare => { if let Err(e) = ensure_sqlx_cli_installed(ui).await { return Err(e.context("Error ensuring sqlx-cli is installed!")); @@ -197,9 +214,6 @@ async fn sync(config: &Config) -> Result { let record = record.context("Failed to read record from CSV file!")?; let station = prepare_station(record, i)?; - // TODO: get name (all languages), country name (all languages), country code, postcode, city (all languages), display name (all languages), street (all languages) - // localhost:8080/reverse?format=jsonv2&lat=48.876742&lon=2.358424&addressdetails=1&namedetails=1 - sqlx::query( r#" INSERT INTO @@ -231,6 +245,49 @@ async fn sync(config: &Config) -> Result { Ok(i) } +async fn augment(config: &Config) -> Result<(u32, u32), anyhow::Error> { + // TODO: get name (all languages), country name (all languages), country code, postcode, city (all languages), display name (all languages), street (all languages) + // localhost:8080/reverse?format=jsonv2&lat=48.876742&lon=2.358424&addressdetails=1&namedetails=1 + + let mut conn = get_db_client(&config.database).await; + let stations = sqlx::query( + r#" + SELECT + id, + latitude, + longitude + FROM + stations; + "#, + ) + .fetch_all(&mut conn) + .await?; + + let client = Client::new(); + + for row in stations { + let id = row.get::(0); + let lat = row.get::(1); + let lon = row.get::(2); + + let data: Value = client + .get(format!("https://nominatim.openstreetmap.org/reverse?format=jsonv2&lat={}&lon={}&namedetails=1&addressdetails=1", lat, lon)) + .header(USER_AGENT, "reStations 0.1") + .send() + .await + .context(format!("Failed to get metadata for station {}!", id))? + .json() + .await + .context(format!( + "Failed to deserialize metadata for station {}!", + id + ))?; + println!("{:?}", data); + } + + Ok((0, 0)) +} + fn get_db_config(config: &DatabaseConfig) -> SqliteConnectOptions { let db_url = Url::parse(&config.url).expect("Invalid DATABASE_URL!"); ConnectOptions::from_url(&db_url).expect("Invalid DATABASE_URL!") @@ -256,17 +313,8 @@ fn prepare_station(record: StringRecord, i: i32) -> Result { + match (id, name, uic, lat, lon, country) { + (Some(id), Some(name), Some(uic), Some(lat), Some(lon), Some(country)) => { let id = id .parse::() .context(format!("Failed to parse ID to i64: {}", id))?;