diff --git a/Cargo.lock b/Cargo.lock index a001175..861af49 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2317,6 +2317,7 @@ dependencies = [ "liquid", "reqwest", "restations-config", + "serde_json", "sqlx", "tempfile", "tokio", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 232a3a8..2c275f0 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -25,7 +25,8 @@ guppy = "0.17" include_dir = "0.7" liquid = "~0.26" restations-config = { path = "../config" } -reqwest = { version = "0.12", features = ["stream"] } +reqwest = { version = "0.12", features = ["stream", "json"] } +serde_json = "1.0" sqlx = { version = "0.8", features = [ "runtime-tokio", "tls-rustls", diff --git a/cli/src/bin/db.rs b/cli/src/bin/db.rs index 31d3284..0b6eb9a 100644 --- a/cli/src/bin/db.rs +++ b/cli/src/bin/db.rs @@ -3,12 +3,13 @@ use clap::{Parser, Subcommand}; use csv_async::{AsyncReaderBuilder, StringRecord, Trim}; use futures::stream::TryStreamExt; use guppy::{Version, VersionReq}; -use reqwest::Client; +use reqwest::{header::USER_AGENT, Client}; use restations_cli::util::ui::UI; use restations_config::DatabaseConfig; use restations_config::{load_config, parse_env, Config, Environment}; +use serde_json::Value; use sqlx::sqlite::{SqliteConnectOptions, SqliteConnection}; -use sqlx::{ConnectOptions, Connection}; +use sqlx::{ConnectOptions, Connection, Row}; use std::path::PathBuf; use std::process::{ExitCode, Stdio}; use tokio::{ @@ -60,6 +61,8 @@ enum Commands { Create, #[command(about = "Synchronize the database with the source data")] Sync, + #[command(about = "Augment the data with additional location data")] + Augment, #[command(about = "Generate query metadata to support offline compile-time verification")] Prepare, } @@ -97,6 +100,20 @@ async fn cli(ui: &mut UI<'_>, cli: Cli) -> Result<(), anyhow::Error> { ui.success(&format!("{} stations synchronized.", stations)); Ok(()) } + Commands::Augment => { + ui.info(&format!("Augmenting {} database…", &cli.env)); + ui.indent(); + let result = augment(&config) + .await + .context("Could not augment database!"); + ui.outdent(); + let (augmented_stations, unavailable_stations) = result?; + ui.success(&format!( + "{} stations augmented, no data found for {} stations.", + augmented_stations, unavailable_stations + )); + Ok(()) + } Commands::Prepare => { if let Err(e) = ensure_sqlx_cli_installed(ui).await { return Err(e.context("Error ensuring sqlx-cli is installed!")); @@ -174,24 +191,6 @@ struct StationRecord { pub latitude: Option, pub longitude: Option, pub country: Option, - pub info_de: Option, - pub info_en: Option, - pub info_es: Option, - pub info_fr: Option, - pub info_it: Option, - pub info_nb: Option, - pub info_nl: Option, - pub info_cs: Option, - pub info_da: Option, - pub info_hu: Option, - pub info_ja: Option, - pub info_ko: Option, - pub info_pl: Option, - pub info_pt: Option, - pub info_ru: Option, - pub info_sv: Option, - pub info_tr: Option, - pub info_zh: Option, } async fn sync(config: &Config) -> Result { @@ -225,28 +224,10 @@ async fn sync(config: &Config) -> Result { uic, latitude, longitude, - country, - info_de, - info_en, - info_es, - info_fr, - info_it, - info_nb, - info_nl, - info_cs, - info_da, - info_hu, - info_ja, - info_ko, - info_pl, - info_pt, - info_ru, - info_sv, - info_tr, - info_zh + country ) VALUES ( - ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ? ) "#, ) @@ -256,24 +237,6 @@ async fn sync(config: &Config) -> Result { .bind(station.latitude) .bind(station.longitude) .bind(station.country) - .bind(station.info_de) - .bind(station.info_en) - .bind(station.info_es) - .bind(station.info_fr) - .bind(station.info_it) - .bind(station.info_nb) - .bind(station.info_nl) - .bind(station.info_cs) - .bind(station.info_da) - .bind(station.info_hu) - .bind(station.info_ja) - .bind(station.info_ko) - .bind(station.info_pl) - .bind(station.info_pt) - .bind(station.info_ru) - .bind(station.info_sv) - .bind(station.info_tr) - .bind(station.info_zh) .execute(&mut conn) .await?; i += 1; @@ -282,6 +245,49 @@ async fn sync(config: &Config) -> Result { Ok(i) } +async fn augment(config: &Config) -> Result<(u32, u32), anyhow::Error> { + // TODO: get name (all languages), country name (all languages), country code, postcode, city (all languages), display name (all languages), street (all languages) + // localhost:8080/reverse?format=jsonv2&lat=48.876742&lon=2.358424&addressdetails=1&namedetails=1 + + let mut conn = get_db_client(&config.database).await; + let stations = sqlx::query( + r#" + SELECT + id, + latitude, + longitude + FROM + stations; + "#, + ) + .fetch_all(&mut conn) + .await?; + + let client = Client::new(); + + for row in stations { + let id = row.get::(0); + let lat = row.get::(1); + let lon = row.get::(2); + + let data: Value = client + .get(format!("https://nominatim.openstreetmap.org/reverse?format=jsonv2&lat={}&lon={}&namedetails=1&addressdetails=1", lat, lon)) + .header(USER_AGENT, "reStations 0.1") + .send() + .await + .context(format!("Failed to get metadata for station {}!", id))? + .json() + .await + .context(format!( + "Failed to deserialize metadata for station {}!", + id + ))?; + println!("{:?}", data); + } + + Ok((0, 0)) +} + fn get_db_config(config: &DatabaseConfig) -> SqliteConnectOptions { let db_url = Url::parse(&config.url).expect("Invalid DATABASE_URL!"); ConnectOptions::from_url(&db_url).expect("Invalid DATABASE_URL!") @@ -306,56 +312,9 @@ fn prepare_station(record: StringRecord, i: i32) -> Result { + + match (id, name, uic, lat, lon, country) { + (Some(id), Some(name), Some(uic), Some(lat), Some(lon), Some(country)) => { let id = id .parse::() .context(format!("Failed to parse ID to i64: {}", id))?; @@ -383,24 +342,6 @@ fn prepare_station(record: StringRecord, i: i32) -> Result Err(anyhow!("Invalid data in line {}!", i)), diff --git a/db/schema.sql b/db/schema.sql index ac78ad6..7154193 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -5,24 +5,7 @@ CREATE TABLE stations ( latitude REAL, longitude REAL, country TEXT, - info_de TEXT, - info_en TEXT, - info_es TEXT, - info_fr TEXT, - info_it TEXT, - info_nb TEXT, - info_nl TEXT, - info_cs TEXT, - info_da TEXT, - info_hu TEXT, - info_ja TEXT, - info_ko TEXT, - info_pl TEXT, - info_pt TEXT, - info_ru TEXT, - info_sv TEXT, - info_tr TEXT, - info_zh TEXT + data JSON ); CREATE UNIQUE INDEX stations_id_idx ON stations (id); diff --git a/db/src/entities/stations.rs b/db/src/entities/stations.rs index 6c83985..4c426be 100644 --- a/db/src/entities/stations.rs +++ b/db/src/entities/stations.rs @@ -20,24 +20,6 @@ pub struct Station { pub latitude: Option, pub longitude: Option, pub country: Option, - pub info_de: Option, - pub info_en: Option, - pub info_es: Option, - pub info_fr: Option, - pub info_it: Option, - pub info_nb: Option, - pub info_nl: Option, - pub info_cs: Option, - pub info_da: Option, - pub info_hu: Option, - pub info_ja: Option, - pub info_ko: Option, - pub info_pl: Option, - pub info_pt: Option, - pub info_ru: Option, - pub info_sv: Option, - pub info_tr: Option, - pub info_zh: Option, } #[derive(Deserialize, Validate, Clone)] @@ -59,24 +41,6 @@ pub struct StationChangeset { pub longitude: Option, #[cfg_attr(feature = "test-helpers", dummy(faker = "CountryName()"))] pub country: Option, - pub info_de: Option, - pub info_en: Option, - pub info_es: Option, - pub info_fr: Option, - pub info_it: Option, - pub info_nb: Option, - pub info_nl: Option, - pub info_cs: Option, - pub info_da: Option, - pub info_hu: Option, - pub info_ja: Option, - pub info_ko: Option, - pub info_pl: Option, - pub info_pt: Option, - pub info_ru: Option, - pub info_sv: Option, - pub info_tr: Option, - pub info_zh: Option, } pub async fn load_all( @@ -90,25 +54,7 @@ pub async fn load_all( uic, latitude, longitude, - country, - info_de, - info_en, - info_es, - info_fr, - info_it, - info_nb, - info_nl, - info_cs, - info_da, - info_hu, - info_ja, - info_ko, - info_pl, - info_pt, - info_ru, - info_sv, - info_tr, - info_zh + country FROM stations" ) @@ -129,25 +75,7 @@ pub async fn load_all_within_limit( uic, latitude, longitude, - country, - info_de, - info_en, - info_es, - info_fr, - info_it, - info_nb, - info_nl, - info_cs, - info_da, - info_hu, - info_ja, - info_ko, - info_pl, - info_pt, - info_ru, - info_sv, - info_tr, - info_zh + country FROM stations LIMIT @@ -171,25 +99,7 @@ pub async fn load( uic, latitude, longitude, - country, - info_de, - info_en, - info_es, - info_fr, - info_it, - info_nb, - info_nl, - info_cs, - info_da, - info_hu, - info_ja, - info_ko, - info_pl, - info_pt, - info_ru, - info_sv, - info_tr, - info_zh + country FROM stations WHERE @@ -219,49 +129,13 @@ pub async fn search_by_name( uic, latitude, longitude, - country, - info_de, - info_en, - info_es, - info_fr, - info_it, - info_nb, - info_nl, - info_cs, - info_da, - info_hu, - info_ja, - info_ko, - info_pl, - info_pt, - info_ru, - info_sv, - info_tr, - info_zh + country FROM stations WHERE uic IS NOT NULL AND ( name LIKE $1 - OR info_de LIKE $1 - OR info_en LIKE $1 - OR info_es LIKE $1 - OR info_fr LIKE $1 - OR info_it LIKE $1 - OR info_nb LIKE $1 - OR info_nl LIKE $1 - OR info_cs LIKE $1 - OR info_da LIKE $1 - OR info_hu LIKE $1 - OR info_ja LIKE $1 - OR info_ko LIKE $1 - OR info_pl LIKE $1 - OR info_pt LIKE $1 - OR info_ru LIKE $1 - OR info_sv LIKE $1 - OR info_tr LIKE $1 - OR info_zh LIKE $1 ) ORDER BY name @@ -298,25 +172,7 @@ pub async fn search_by_position( uic, latitude, longitude, - country, - info_de, - info_en, - info_es, - info_fr, - info_it, - info_nb, - info_nl, - info_cs, - info_da, - info_hu, - info_ja, - info_ko, - info_pl, - info_pt, - info_ru, - info_sv, - info_tr, - info_zh + country FROM stations WHERE @@ -367,25 +223,7 @@ pub async fn search_by_name_and_position( uic, latitude, longitude, - country, - info_de, - info_en, - info_es, - info_fr, - info_it, - info_nb, - info_nl, - info_cs, - info_da, - info_hu, - info_ja, - info_ko, - info_pl, - info_pt, - info_ru, - info_sv, - info_tr, - info_zh + country FROM stations WHERE diff --git a/db/src/test_helpers/stations.rs b/db/src/test_helpers/stations.rs index 2204769..3652585 100644 --- a/db/src/test_helpers/stations.rs +++ b/db/src/test_helpers/stations.rs @@ -8,31 +8,13 @@ pub async fn create(station: StationChangeset, db: &DbPool) -> Result Result