Skip to content

Addtl. place data #117

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
marcoow wants to merge 5 commits into
base: main
Choose a base branch
from
Draft

Addtl. place data #117

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5301dbe
remove info_xy columns
marcoow Apr 2, 2025
5edbb69
prepare nominatim import
marcoow Apr 2, 2025
6ad989d
add JSON data column
marcoow Apr 2, 2025
23c1123
load all of Europe into Nominatim
marcoow Apr 2, 2025
7b7d181
get metadata per station
marcoow Apr 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion cli/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ guppy = "0.17"
include_dir = "0.7"
liquid = "~0.26"
restations-config = { path = "../config" }
reqwest = { version = "0.12", features = ["stream"] }
reqwest = { version = "0.12", features = ["stream", "json"] }
serde_json = "1.0"
sqlx = { version = "0.8", features = [
"runtime-tokio",
"tls-rustls",
Expand Down
193 changes: 67 additions & 126 deletions cli/src/bin/db.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@ use clap::{Parser, Subcommand};
use csv_async::{AsyncReaderBuilder, StringRecord, Trim};
use futures::stream::TryStreamExt;
use guppy::{Version, VersionReq};
use reqwest::Client;
use reqwest::{header::USER_AGENT, Client};
use restations_cli::util::ui::UI;
use restations_config::DatabaseConfig;
use restations_config::{load_config, parse_env, Config, Environment};
use serde_json::Value;
use sqlx::sqlite::{SqliteConnectOptions, SqliteConnection};
use sqlx::{ConnectOptions, Connection};
use sqlx::{ConnectOptions, Connection, Row};
use std::path::PathBuf;
use std::process::{ExitCode, Stdio};
use tokio::{
Expand Down Expand Up @@ -60,6 +61,8 @@ enum Commands {
Create,
#[command(about = "Synchronize the database with the source data")]
Sync,
#[command(about = "Augment the data with additional location data")]
Augment,
#[command(about = "Generate query metadata to support offline compile-time verification")]
Prepare,
}
Expand Down Expand Up @@ -97,6 +100,20 @@ async fn cli(ui: &mut UI<'_>, cli: Cli) -> Result<(), anyhow::Error> {
ui.success(&format!("{} stations synchronized.", stations));
Ok(())
}
Commands::Augment => {
ui.info(&format!("Augmenting {} database…", &cli.env));
ui.indent();
let result = augment(&config)
.await
.context("Could not augment database!");
ui.outdent();
let (augmented_stations, unavailable_stations) = result?;
ui.success(&format!(
"{} stations augmented, no data found for {} stations.",
augmented_stations, unavailable_stations
));
Ok(())
}
Commands::Prepare => {
if let Err(e) = ensure_sqlx_cli_installed(ui).await {
return Err(e.context("Error ensuring sqlx-cli is installed!"));
Expand Down Expand Up @@ -174,24 +191,6 @@ struct StationRecord {
pub latitude: Option<f64>,
pub longitude: Option<f64>,
pub country: Option<String>,
pub info_de: Option<String>,
pub info_en: Option<String>,
pub info_es: Option<String>,
pub info_fr: Option<String>,
pub info_it: Option<String>,
pub info_nb: Option<String>,
pub info_nl: Option<String>,
pub info_cs: Option<String>,
pub info_da: Option<String>,
pub info_hu: Option<String>,
pub info_ja: Option<String>,
pub info_ko: Option<String>,
pub info_pl: Option<String>,
pub info_pt: Option<String>,
pub info_ru: Option<String>,
pub info_sv: Option<String>,
pub info_tr: Option<String>,
pub info_zh: Option<String>,
}

async fn sync(config: &Config) -> Result<i32, anyhow::Error> {
Expand Down Expand Up @@ -225,28 +224,10 @@ async fn sync(config: &Config) -> Result<i32, anyhow::Error> {
uic,
latitude,
longitude,
country,
info_de,
info_en,
info_es,
info_fr,
info_it,
info_nb,
info_nl,
info_cs,
info_da,
info_hu,
info_ja,
info_ko,
info_pl,
info_pt,
info_ru,
info_sv,
info_tr,
info_zh
country
)
VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
?, ?, ?, ?, ?, ?
)
"#,
)
Expand All @@ -256,24 +237,6 @@ async fn sync(config: &Config) -> Result<i32, anyhow::Error> {
.bind(station.latitude)
.bind(station.longitude)
.bind(station.country)
.bind(station.info_de)
.bind(station.info_en)
.bind(station.info_es)
.bind(station.info_fr)
.bind(station.info_it)
.bind(station.info_nb)
.bind(station.info_nl)
.bind(station.info_cs)
.bind(station.info_da)
.bind(station.info_hu)
.bind(station.info_ja)
.bind(station.info_ko)
.bind(station.info_pl)
.bind(station.info_pt)
.bind(station.info_ru)
.bind(station.info_sv)
.bind(station.info_tr)
.bind(station.info_zh)
.execute(&mut conn)
.await?;
i += 1;
Expand All @@ -282,6 +245,49 @@ async fn sync(config: &Config) -> Result<i32, anyhow::Error> {
Ok(i)
}

async fn augment(config: &Config) -> Result<(u32, u32), anyhow::Error> {
// TODO: get name (all languages), country name (all languages), country code, postcode, city (all languages), display name (all languages), street (all languages)
// localhost:8080/reverse?format=jsonv2&lat=48.876742&lon=2.358424&addressdetails=1&namedetails=1

let mut conn = get_db_client(&config.database).await;
let stations = sqlx::query(
r#"
SELECT
id,
latitude,
longitude
FROM
stations;
"#,
)
.fetch_all(&mut conn)
.await?;

let client = Client::new();

for row in stations {
let id = row.get::<i64, usize>(0);
let lat = row.get::<f64, usize>(1);
let lon = row.get::<f64, usize>(2);

let data: Value = client
.get(format!("https://nominatim.openstreetmap.org/reverse?format=jsonv2&lat={}&lon={}&namedetails=1&addressdetails=1", lat, lon))
.header(USER_AGENT, "reStations 0.1")
.send()
.await
.context(format!("Failed to get metadata for station {}!", id))?
.json()
.await
.context(format!(
"Failed to deserialize metadata for station {}!",
id
))?;
println!("{:?}", data);
}

Ok((0, 0))
}

fn get_db_config(config: &DatabaseConfig) -> SqliteConnectOptions {
let db_url = Url::parse(&config.url).expect("Invalid DATABASE_URL!");
ConnectOptions::from_url(&db_url).expect("Invalid DATABASE_URL!")
Expand All @@ -306,56 +312,9 @@ fn prepare_station(record: StringRecord, i: i32) -> Result<StationRecord, anyhow
let lat = record.get(5);
let lon = record.get(6);
let country = record.get(8);
let info_de = record.get(54);
let info_en = record.get(55);
let info_es = record.get(56);
let info_fr = record.get(57);
let info_it = record.get(58);
let info_nb = record.get(59);
let info_nl = record.get(60);
let info_cs = record.get(61);
let info_da = record.get(62);
let info_hu = record.get(63);
let info_ja = record.get(64);
let info_ko = record.get(65);
let info_pl = record.get(66);
let info_pt = record.get(67);
let info_ru = record.get(68);
let info_sv = record.get(69);
let info_tr = record.get(70);
let info_zh = record.get(71);

match (
id, name, uic, lat, lon, country, info_de, info_en, info_es, info_fr, info_it, info_nb,
info_nl, info_cs, info_da, info_hu, info_ja, info_ko, info_pl, info_pt, info_ru, info_sv,
info_tr, info_zh,
) {
(
Some(id),
Some(name),
Some(uic),
Some(lat),
Some(lon),
Some(country),
Some(info_de),
Some(info_en),
Some(info_es),
Some(info_fr),
Some(info_it),
Some(info_nb),
Some(info_nl),
Some(info_cs),
Some(info_da),
Some(info_hu),
Some(info_ja),
Some(info_ko),
Some(info_pl),
Some(info_pt),
Some(info_ru),
Some(info_sv),
Some(info_tr),
Some(info_zh),
) => {

match (id, name, uic, lat, lon, country) {
(Some(id), Some(name), Some(uic), Some(lat), Some(lon), Some(country)) => {
let id = id
.parse::<i64>()
.context(format!("Failed to parse ID to i64: {}", id))?;
Expand Down Expand Up @@ -383,24 +342,6 @@ fn prepare_station(record: StringRecord, i: i32) -> Result<StationRecord, anyhow
latitude: lat,
longitude: lon,
country: prepare_csv_string(country),
info_de: prepare_csv_string(info_de),
info_en: prepare_csv_string(info_en),
info_es: prepare_csv_string(info_es),
info_fr: prepare_csv_string(info_fr),
info_it: prepare_csv_string(info_it),
info_nb: prepare_csv_string(info_nb),
info_nl: prepare_csv_string(info_nl),
info_cs: prepare_csv_string(info_cs),
info_da: prepare_csv_string(info_da),
info_hu: prepare_csv_string(info_hu),
info_ja: prepare_csv_string(info_ja),
info_ko: prepare_csv_string(info_ko),
info_pl: prepare_csv_string(info_pl),
info_pt: prepare_csv_string(info_pt),
info_ru: prepare_csv_string(info_ru),
info_sv: prepare_csv_string(info_sv),
info_tr: prepare_csv_string(info_tr),
info_zh: prepare_csv_string(info_zh),
})
}
_ => Err(anyhow!("Invalid data in line {}!", i)),
Expand Down
19 changes: 1 addition & 18 deletions db/schema.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,24 +5,7 @@ CREATE TABLE stations (
latitude REAL,
longitude REAL,
country TEXT,
info_de TEXT,
info_en TEXT,
info_es TEXT,
info_fr TEXT,
info_it TEXT,
info_nb TEXT,
info_nl TEXT,
info_cs TEXT,
info_da TEXT,
info_hu TEXT,
info_ja TEXT,
info_ko TEXT,
info_pl TEXT,
info_pt TEXT,
info_ru TEXT,
info_sv TEXT,
info_tr TEXT,
info_zh TEXT
data JSON
);

CREATE UNIQUE INDEX stations_id_idx ON stations (id);
Loading