From 47105ed634270578055cea2803cd6888c5a8eb6d Mon Sep 17 00:00:00 2001 From: Keith Lohnes Date: Fri, 11 Apr 2025 19:03:15 -0400 Subject: [PATCH 01/19] Add template args and parameterize migrations --- Cargo.lock | 11 ++ sqlx-core/Cargo.toml | 1 + sqlx-core/src/migrate/error.rs | 3 + sqlx-core/src/migrate/migration.rs | 186 ++++++++++++++++++++++++++++- sqlx-core/src/migrate/migrator.rs | 23 ++++ 5 files changed, 222 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e19b2e33c8..1a09e18e5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3466,6 +3466,7 @@ dependencies = [ "sha2", "smallvec", "sqlx", + "subst", "thiserror 2.0.11", "time", "tokio", @@ -3828,6 +3829,16 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "subst" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33e7942675ea19db01ef8cf15a1e6443007208e6c74568bd64162da26d40160d" +dependencies = [ + "memchr", + "unicode-width 0.1.14", +] + [[package]] name = "subtle" version = "2.6.1" diff --git a/sqlx-core/Cargo.toml b/sqlx-core/Cargo.toml index f6017a9fee..d162415419 100644 --- a/sqlx-core/Cargo.toml +++ b/sqlx-core/Cargo.toml @@ -83,6 +83,7 @@ hashlink = "0.10.0" indexmap = "2.0" event-listener = "5.2.0" hashbrown = "0.15.0" +subst = "0.3.7" [dev-dependencies] sqlx = { workspace = true, features = ["postgres", "sqlite", "mysql", "migrate", "macros", "time", "uuid"] } diff --git a/sqlx-core/src/migrate/error.rs b/sqlx-core/src/migrate/error.rs index 608d55b18d..68c0895321 100644 --- a/sqlx-core/src/migrate/error.rs +++ b/sqlx-core/src/migrate/error.rs @@ -39,4 +39,7 @@ pub enum MigrateError { "migration {0} is partially applied; fix and remove row from `_sqlx_migrations` table" )] Dirty(i64), + + #[error("migration {0} was missing a parameter")] + MissingParameter(String), } diff --git a/sqlx-core/src/migrate/migration.rs b/sqlx-core/src/migrate/migration.rs index 9bd7f569d8..c9838e0e42 100644 --- a/sqlx-core/src/migrate/migration.rs +++ b/sqlx-core/src/migrate/migration.rs @@ -1,8 +1,12 @@ use std::borrow::Cow; +use std::collections::HashMap; use sha2::{Digest, Sha384}; -use super::MigrationType; +use super::{MigrateError, MigrationType}; + +const ENABLE_SUBSTITUTION: &str = "-- enable-substitution"; +const DISABLE_SUBSTITUTION: &str = "-- disable-substitution"; #[derive(Debug, Clone)] pub struct Migration { @@ -23,7 +27,6 @@ impl Migration { no_tx: bool, ) -> Self { let checksum = Cow::Owned(Vec::from(Sha384::digest(sql.as_bytes()).as_slice())); - Migration { version, description, @@ -33,6 +36,56 @@ impl Migration { no_tx, } } + + pub fn process_parameters( + &self, + params: &HashMap, + ) -> Result { + let Migration { + version, + description, + migration_type, + sql, + checksum, + no_tx, + } = self; + + let mut new_sql = String::with_capacity(sql.len()); + let mut substitution_enabled = false; + + for (i, line) in sql.lines().enumerate() { + if i != 0 { + new_sql.push('\n') + } + let trimmed_line = line.trim(); + if trimmed_line == ENABLE_SUBSTITUTION { + substitution_enabled = true; + new_sql.push_str(line); + continue; + } else if trimmed_line == DISABLE_SUBSTITUTION { + new_sql.push_str(line); + substitution_enabled = false; + continue; + } + + if substitution_enabled { + let substituted_line = subst::substitute(line, params) + .map_err(|e| MigrateError::MissingParameter(e.to_string()))?; + new_sql.push_str(&substituted_line); + } else { + new_sql.push_str(line); + } + } + + Ok(Migration { + version: *version, + description: description.clone(), + migration_type: *migration_type, + sql: Cow::Owned(new_sql), + checksum: checksum.clone(), + no_tx: *no_tx, + }) + } } #[derive(Debug, Clone)] @@ -40,3 +93,132 @@ pub struct AppliedMigration { pub version: i64, pub checksum: Cow<'static, [u8]>, } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_migration_process_parameters_with_substitution() -> Result<(), MigrateError> { + const CREATE_USER: &str = r#" + -- enable-substitution + CREATE USER '${substitution_test_user}'; + -- disable-substitution + CREATE TABLE foo ( + id BIG SERIAL PRIMARY KEY + foo TEXT + ); + -- enable-substitution + DROP USER '${substitution_test_user}'; + -- disable-substitution + "#; + const EXPECTED_RESULT: &str = r#" + -- enable-substitution + CREATE USER 'my_user'; + -- disable-substitution + CREATE TABLE foo ( + id BIG SERIAL PRIMARY KEY + foo TEXT + ); + -- enable-substitution + DROP USER 'my_user'; + -- disable-substitution + "#; + + let migration = Migration::new( + 1, + Cow::Owned("test a simple parameter substitution".to_string()), + crate::migrate::MigrationType::Simple, + Cow::Owned(CREATE_USER.to_string()), + true, + ); + let result = migration.process_parameters(&HashMap::from([( + String::from("substitution_test_user"), + String::from("my_user"), + )]))?; + assert_eq!(result.sql, EXPECTED_RESULT); + Ok(()) + } + + #[test] + fn test_migration_process_parameters_no_substitution() -> Result<(), MigrateError> { + const CREATE_TABLE: &str = r#" + CREATE TABLE foo ( + id BIG SERIAL PRIMARY KEY + foo TEXT + ); + "#; + let migration = Migration::new( + 1, + std::borrow::Cow::Owned("test a simple parameter substitution".to_string()), + crate::migrate::MigrationType::Simple, + Cow::Owned(CREATE_TABLE.to_string()), + true, + ); + let result = migration.process_parameters(&HashMap::from([( + String::from("substitution_test_user"), + String::from("my_user"), + )]))?; + assert_eq!(result.sql, CREATE_TABLE); + Ok(()) + } + + #[test] + fn test_migration_process_parameters_missing_key() -> Result<(), MigrateError> { + const CREATE_TABLE: &str = r#" + -- enable-substitution + CREATE TABLE foo ( + id BIG SERIAL PRIMARY KEY + foo TEXT, + field ${TEST_MISSING_KEY} + ); + -- disable-substitution + + "#; + let migration = Migration::new( + 1, + Cow::Owned("test a simple parameter substitution".to_string()), + crate::migrate::MigrationType::Simple, + Cow::Owned(CREATE_TABLE.to_string()), + true, + ); + let Err(MigrateError::MissingParameter(_)) = + migration.process_parameters(&HashMap::with_capacity(0)) + else { + panic!("Missing env var not caught in process parameters missing env var") + }; + Ok(()) + } + + #[test] + fn test_migration_process_parameters_missing_key_with_default_value() -> Result<(), MigrateError> { + const CREATE_TABLE: &str = r#" + -- enable-substitution + CREATE TABLE foo ( + id BIG SERIAL PRIMARY KEY + foo TEXT, + field ${TEST_MISSING_KEY:TEXT} + ); + -- disable-substitution + "#; + const EXPECTED_CREATE_TABLE: &str = r#" + -- enable-substitution + CREATE TABLE foo ( + id BIG SERIAL PRIMARY KEY + foo TEXT, + field TEXT + ); + -- disable-substitution + "#; + let migration = Migration::new( + 1, + Cow::Owned("test a simple parameter substitution".to_string()), + crate::migrate::MigrationType::Simple, + Cow::Owned(CREATE_TABLE.to_string()), + true, + ); + let result = migration.process_parameters(&HashMap::with_capacity(0))?; + assert_eq!(result.sql, EXPECTED_CREATE_TABLE); + Ok(()) + } +} diff --git a/sqlx-core/src/migrate/migrator.rs b/sqlx-core/src/migrate/migrator.rs index 3209ba6e45..f7daa42c58 100644 --- a/sqlx-core/src/migrate/migrator.rs +++ b/sqlx-core/src/migrate/migrator.rs @@ -23,6 +23,8 @@ pub struct Migrator { pub locking: bool, #[doc(hidden)] pub no_tx: bool, + #[doc(hidden)] + pub template_args: Option>, } fn validate_applied_migrations( @@ -51,8 +53,29 @@ impl Migrator { ignore_missing: false, no_tx: false, locking: true, + template_args: None, }; + /// Set or update template arguments for migration placeholders. + /// + /// # Examples + /// + /// ```rust + /// # use sqlx_core::migrate::Migrator; + /// let mut migrator = Migrator::DEFAULT; + /// migrator.set_template_args(vec![("key", "value"), ("name", "test")]); + /// ``` + pub fn set_template_args(&mut self, args: I) -> &Self + where + I: IntoIterator, + K: Into, + V: Into, + { + let map: HashMap = args.into_iter().map(|(k, v)| (k.into(), v.into())).collect(); + self.template_args = Some(map); + self + } + /// Creates a new instance with the given source. /// /// # Examples From cc6dfed6d524211ed012d01231b81dd7073182a0 Mon Sep 17 00:00:00 2001 From: Keith Lohnes Date: Mon, 14 Apr 2025 15:48:27 -0400 Subject: [PATCH 02/19] Process parameters when running migrations. Make naming more consistent This commit adds the actual processing of template parameters for migrations. It also switches over `args` for `parameters` to be more in line with the nomenclature in the issue --- sqlx-core/src/migrate/migrator.rs | 36 +++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/sqlx-core/src/migrate/migrator.rs b/sqlx-core/src/migrate/migrator.rs index f7daa42c58..fa2cf62285 100644 --- a/sqlx-core/src/migrate/migrator.rs +++ b/sqlx-core/src/migrate/migrator.rs @@ -24,7 +24,9 @@ pub struct Migrator { #[doc(hidden)] pub no_tx: bool, #[doc(hidden)] - pub template_args: Option>, + pub template_params: Option>, + #[doc(hidden)] + pub template_parameters_from_env: bool, } fn validate_applied_migrations( @@ -53,26 +55,27 @@ impl Migrator { ignore_missing: false, no_tx: false, locking: true, - template_args: None, + template_params: None, + template_parameters_from_env: false, }; - /// Set or update template arguments for migration placeholders. + /// Set or update template parameters for migration placeholders. /// /// # Examples /// /// ```rust /// # use sqlx_core::migrate::Migrator; /// let mut migrator = Migrator::DEFAULT; - /// migrator.set_template_args(vec![("key", "value"), ("name", "test")]); + /// migrator.set_template_parameters(vec![("key", "value"), ("name", "test")]); /// ``` - pub fn set_template_args(&mut self, args: I) -> &Self + pub fn set_template_parameters(&mut self, params: I) -> &Self where I: IntoIterator, K: Into, V: Into, { - let map: HashMap = args.into_iter().map(|(k, v)| (k.into(), v.into())).collect(); - self.template_args = Some(map); + let map: HashMap = params.into_iter().map(|(k, v)| (k.into(), v.into())).collect(); + self.template_params = Some(map); self } @@ -110,6 +113,12 @@ impl Migrator { self } + /// Specify whether template parameters for migrations should be read from the environment + pub fn set_template_parameters_from_env(&mut self, template_paramaters_from_env: bool) -> &Self { + self.template_parameters_from_env = template_paramaters_from_env; + self + } + /// Specify whether or not to lock the database during migration. Defaults to `true`. /// /// ### Warning @@ -188,6 +197,14 @@ impl Migrator { .map(|m| (m.version, m)) .collect(); + + //Bind so we're not collecting for each migration + let env_params = if self.template_parameters_from_env { + Some(std::env::vars().collect()) + } else { + None + }; + for migration in self.iter() { if migration.migration_type.is_down_migration() { continue; @@ -200,6 +217,11 @@ impl Migrator { } } None => { + if self.template_parameters_from_env { + migration.process_parameters(env_params.as_ref().unwrap())?; + } else if let Some(params) = self.template_params.as_ref() { + migration.process_parameters(params)?; + } conn.apply(migration).await?; } } From 68312626566e5411f6c3d744df80945c957dee1a Mon Sep 17 00:00:00 2001 From: Keith Lohnes Date: Mon, 14 Apr 2025 16:59:56 -0400 Subject: [PATCH 03/19] Add parameter functionality to sqlx-cli --- sqlx-cli/src/database.rs | 2 +- sqlx-cli/src/lib.rs | 4 ++++ sqlx-cli/src/migrate.rs | 17 +++++++++++++++++ sqlx-cli/src/opt.rs | 24 ++++++++++++++++++++++++ sqlx-core/src/migrate/migrator.rs | 1 - 5 files changed, 46 insertions(+), 2 deletions(-) diff --git a/sqlx-cli/src/database.rs b/sqlx-cli/src/database.rs index 7a9bc6bf2f..171721b776 100644 --- a/sqlx-cli/src/database.rs +++ b/sqlx-cli/src/database.rs @@ -57,7 +57,7 @@ pub async fn reset( pub async fn setup(migration_source: &str, connect_opts: &ConnectOpts) -> anyhow::Result<()> { create(connect_opts).await?; - migrate::run(migration_source, connect_opts, false, false, None).await + migrate::run(migration_source, connect_opts, false, false, None, false, Vec::with_capacity(0)).await } async fn ask_to_continue_drop(db_url: String) -> bool { diff --git a/sqlx-cli/src/lib.rs b/sqlx-cli/src/lib.rs index cb31205b4f..dee8c948dc 100644 --- a/sqlx-cli/src/lib.rs +++ b/sqlx-cli/src/lib.rs @@ -66,6 +66,8 @@ async fn do_run(opt: Opt) -> Result<()> { ignore_missing, connect_opts, target_version, + params_from_env, + parameters, } => { migrate::run( &source, @@ -73,6 +75,8 @@ async fn do_run(opt: Opt) -> Result<()> { dry_run, *ignore_missing, target_version, + params_from_env, + parameters, ) .await? } diff --git a/sqlx-cli/src/migrate.rs b/sqlx-cli/src/migrate.rs index e00f6de651..d15a9b437e 100644 --- a/sqlx-cli/src/migrate.rs +++ b/sqlx-cli/src/migrate.rs @@ -277,7 +277,11 @@ pub async fn run( dry_run: bool, ignore_missing: bool, target_version: Option, + params_from_env: bool, + parameters: Vec<(String, String)>, ) -> anyhow::Result<()> { + + println!("{:#?}", parameters); let migrator = Migrator::new(Path::new(migration_source)).await?; if let Some(target_version) = target_version { if !migrator.version_exists(target_version) { @@ -313,6 +317,14 @@ pub async fn run( .map(|m| (m.version, m)) .collect(); + let env_params: HashMap<_, _> = if params_from_env { + std::env::vars().collect() + } else { + HashMap::with_capacity(0) + }; + + let params: HashMap<_, _> = parameters.into_iter().collect(); + for migration in migrator.iter() { if migration.migration_type.is_down_migration() { // Skipping down migrations @@ -332,6 +344,11 @@ pub async fn run( let elapsed = if dry_run || skip { Duration::new(0, 0) } else { + if params_from_env { + migration.process_parameters(&env_params)?; + } else if !params.is_empty() { + migration.process_parameters(¶ms)?; + } conn.apply(migration).await? }; let text = if skip { diff --git a/sqlx-cli/src/opt.rs b/sqlx-cli/src/opt.rs index a0e19e47f4..d2f8030355 100644 --- a/sqlx-cli/src/opt.rs +++ b/sqlx-cli/src/opt.rs @@ -1,3 +1,4 @@ +use std::error::Error; use std::ops::{Deref, Not}; use clap::{Args, Parser}; @@ -181,6 +182,15 @@ pub enum MigrateCommand { /// pending migrations. If already at the target version, then no-op. #[clap(long)] target_version: Option, + + #[clap(long)] + /// Template parameters for substitution in migrations from environment variables + params_from_env: bool, + + #[clap(long, short, value_parser = parse_key_val::, num_args = 1, value_delimiter=',')] + /// Provide template parameters for substitution in migrations, e.g. --parameters + /// key:value,key2:value2 + parameters: Vec<(String, String)>, }, /// Revert the latest migration with a down file. @@ -325,3 +335,17 @@ impl Not for IgnoreMissing { !self.ignore_missing } } + +/// Parse a single key-value pair +fn parse_key_val(s: &str) -> Result<(T, U), Box> +where + T: std::str::FromStr, + T::Err: Error + Send + Sync + 'static, + U: std::str::FromStr, + U::Err: Error + Send + Sync + 'static, +{ + let pos = s + .find('=') + .ok_or_else(|| format!("invalid KEY=value: no `=` found in `{s}`"))?; + Ok((s[..pos].parse()?, s[pos + 1..].parse()?)) +} diff --git a/sqlx-core/src/migrate/migrator.rs b/sqlx-core/src/migrate/migrator.rs index fa2cf62285..75c1820758 100644 --- a/sqlx-core/src/migrate/migrator.rs +++ b/sqlx-core/src/migrate/migrator.rs @@ -198,7 +198,6 @@ impl Migrator { .collect(); - //Bind so we're not collecting for each migration let env_params = if self.template_parameters_from_env { Some(std::env::vars().collect()) } else { From 1d3eca13d7647cf208def380b065d65446d77485 Mon Sep 17 00:00:00 2001 From: Keith Lohnes Date: Wed, 23 Apr 2025 17:01:18 -0400 Subject: [PATCH 04/19] Add compile time checking of missing parameters --- Cargo.lock | 6 +- Cargo.toml | 1 + sqlx-core/Cargo.toml | 2 +- sqlx-macros-core/Cargo.toml | 1 + sqlx-macros-core/src/migrate.rs | 39 ++++++-- sqlx-macros-core/src/test_attr.rs | 4 +- sqlx-macros/Cargo.toml | 1 + sqlx-macros/src/lib.rs | 93 ++++++++++++++++++- src/macros/mod.rs | 20 ++-- tests/ui-tests.rs | 7 ++ tests/ui/migrate/invalid_key.rs | 4 + tests/ui/migrate/invalid_key.stderr | 8 ++ .../20250423195520_create_users.down.sql | 1 + .../20250423195520_create_users.up.sql | 4 + tests/ui/migrate/missing_parameter.rs | 5 + tests/ui/migrate/missing_parameter.stderr | 8 ++ 16 files changed, 184 insertions(+), 20 deletions(-) create mode 100644 tests/ui/migrate/invalid_key.rs create mode 100644 tests/ui/migrate/invalid_key.stderr create mode 100644 tests/ui/migrate/migrations/20250423195520_create_users.down.sql create mode 100644 tests/ui/migrate/migrations/20250423195520_create_users.up.sql create mode 100644 tests/ui/migrate/missing_parameter.rs create mode 100644 tests/ui/migrate/missing_parameter.stderr diff --git a/Cargo.lock b/Cargo.lock index 1a09e18e5d..0ce819e66a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2754,9 +2754,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] @@ -3611,6 +3611,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", + "subst", "syn 2.0.96", ] @@ -3633,6 +3634,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", + "subst", "syn 2.0.96", "tempfile", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 1aef121199..2c15f55d30 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -155,6 +155,7 @@ uuid = "1.1.2" # Common utility crates dotenvy = { version = "0.15.7", default-features = false } +subst = "0.3.7" # Runtimes [workspace.dependencies.async-std] diff --git a/sqlx-core/Cargo.toml b/sqlx-core/Cargo.toml index d162415419..b1f1fd8dad 100644 --- a/sqlx-core/Cargo.toml +++ b/sqlx-core/Cargo.toml @@ -83,7 +83,7 @@ hashlink = "0.10.0" indexmap = "2.0" event-listener = "5.2.0" hashbrown = "0.15.0" -subst = "0.3.7" +subst = { workspace = true } [dev-dependencies] sqlx = { workspace = true, features = ["postgres", "sqlite", "mysql", "migrate", "macros", "time", "uuid"] } diff --git a/sqlx-macros-core/Cargo.toml b/sqlx-macros-core/Cargo.toml index 85efa80912..be2bd87b1d 100644 --- a/sqlx-macros-core/Cargo.toml +++ b/sqlx-macros-core/Cargo.toml @@ -64,6 +64,7 @@ proc-macro2 = { version = "1.0.79", default-features = false } serde = { version = "1.0.132", features = ["derive"] } serde_json = { version = "1.0.73" } sha2 = { version = "0.10.0" } +subst = { workspace = true } syn = { version = "2.0.52", default-features = false, features = ["full", "derive", "parsing", "printing", "clone-impls"] } tempfile = { version = "3.10.1" } quote = { version = "1.0.26", default-features = false } diff --git a/sqlx-macros-core/src/migrate.rs b/sqlx-macros-core/src/migrate.rs index c9cf5b8eb1..ba57499dc0 100644 --- a/sqlx-macros-core/src/migrate.rs +++ b/sqlx-macros-core/src/migrate.rs @@ -1,6 +1,7 @@ #[cfg(any(sqlx_macros_unstable, procmacro2_semver_exempt))] extern crate proc_macro; +use std::collections::HashMap; use std::path::{Path, PathBuf}; use proc_macro2::TokenStream; @@ -81,20 +82,28 @@ impl ToTokens for QuoteMigration { } } -pub fn expand_migrator_from_lit_dir(dir: LitStr) -> crate::Result { - expand_migrator_from_dir(&dir.value(), dir.span()) +pub fn expand_migrator_from_lit_dir( + dir: LitStr, + parameters: Option>, +) -> crate::Result { + expand_migrator_from_dir(&dir.value(), dir.span(), parameters) } pub(crate) fn expand_migrator_from_dir( dir: &str, err_span: proc_macro2::Span, + parameters: Option>, ) -> crate::Result { let path = crate::common::resolve_path(dir, err_span)?; - - expand_migrator(&path) + expand_migrator(&path, parameters) } -pub(crate) fn expand_migrator(path: &Path) -> crate::Result { +pub(crate) fn expand_migrator( + path: &Path, + parameters: Option>, +) -> crate::Result { + const ENABLE_SUBSTITUTION: &str = "-- enable-substitution"; + const DISABLE_SUBSTITUTION: &str = "-- disable-substitution"; let path = path.canonicalize().map_err(|e| { format!( "error canonicalizing migration directory {}: {e}", @@ -103,9 +112,27 @@ pub(crate) fn expand_migrator(path: &Path) -> crate::Result { })?; // Use the same code path to resolve migrations at compile time and runtime. + let mut substitution_enabled = false; let migrations = sqlx_core::migrate::resolve_blocking(&path)? .into_iter() - .map(|(migration, path)| QuoteMigration { migration, path }); + .map(|(migration, path)| { + if let Some(ref params) = parameters { + for line in migration.sql.lines() { + let trimmed_line = line.trim(); + if trimmed_line == ENABLE_SUBSTITUTION { + substitution_enabled = true; + continue; + } else if trimmed_line == DISABLE_SUBSTITUTION { + substitution_enabled = false; + continue; + } + if substitution_enabled { + subst::substitute(line, params).expect("Missing substitution parameter"); + } + } + } + QuoteMigration { migration, path } + }); #[cfg(any(sqlx_macros_unstable, procmacro2_semver_exempt))] { diff --git a/sqlx-macros-core/src/test_attr.rs b/sqlx-macros-core/src/test_attr.rs index d7c6eb0486..ee70d17fa5 100644 --- a/sqlx-macros-core/src/test_attr.rs +++ b/sqlx-macros-core/src/test_attr.rs @@ -143,7 +143,7 @@ fn expand_advanced(args: AttributeArgs, input: syn::ItemFn) -> crate::Result { - let migrator = crate::migrate::expand_migrator_from_lit_dir(path)?; + let migrator = crate::migrate::expand_migrator_from_lit_dir(path, None)?; quote! { args.migrator(&#migrator); } } MigrationsOpt::InferredPath if !inputs.is_empty() => { @@ -151,7 +151,7 @@ fn expand_advanced(args: AttributeArgs, input: syn::ItemFn) -> crate::Result TokenStream { #[cfg(feature = "migrate")] #[proc_macro] pub fn migrate(input: TokenStream) -> TokenStream { - use syn::LitStr; + use std::collections::HashMap; + use syn::{parse_macro_input, Expr, ExprArray, ExprLit, ExprPath, ExprTuple, Lit, LitStr}; - let input = syn::parse_macro_input!(input as LitStr); - match migrate::expand_migrator_from_lit_dir(input) { + // Extract directory path, handling both direct literals and grouped literals + fn extract_dir(expr: Option) -> LitStr { + if let Some(Expr::Lit(ExprLit { + lit: Lit::Str(lit_str), + .. + })) = expr + { + return lit_str; + } else if let Some(Expr::Group(group)) = expr { + if let Expr::Lit(ExprLit { + lit: Lit::Str(lit_str), + .. + }) = *group.expr + { + return lit_str; + } + } + panic!("Expected a string literal for the directory path."); + } + + // Extract a `String` value from an expression (either a string literal or a variable) + fn extract_value(expr: Expr, location: &str) -> String { + match expr { + Expr::Lit(ExprLit { + lit: Lit::Str(lit_str), + .. + }) => lit_str.value(), + Expr::Path(ExprPath { path, .. }) => path.segments.last().unwrap().ident.to_string(), + _ => panic!("Expected a string literal or a variable in {location}"), + } + } + + // Parse substitutions, expecting an array of tuples (String, Expr) + fn parse_substitutions(expr: Option) -> Option> { + let Expr::Group(group) = expr? else { + return None; + }; + let Expr::Array(ExprArray { elems, .. }) = *group.expr else { + panic!("Expected an array of tuples (String, Expr)."); + }; + + let mut map = HashMap::new(); + for elem in elems { + let Expr::Tuple(ExprTuple { + elems: tuple_elems, .. + }) = elem + else { + panic!("Expected a tuple (String, Expr). Got {:#?}", elem); + }; + + let mut tuple_elems = tuple_elems.into_iter(); + + let key = extract_value(tuple_elems.next().expect("Missing key in tuple."), "key"); + let value = extract_value( + tuple_elems.next().expect("Missing value in tuple."), + "value", + ); + map.insert(key, value); + } + Some(map) + } + + // Parse input and extract directory and optional parameters + let exp = parse_macro_input!(input as syn::Expr); + let (dir, parameters) = match exp { + Expr::Tuple(ExprTuple { elems, .. }) => { + let mut elems = elems.into_iter(); + (extract_dir(elems.next()), elems.next()) + } + Expr::Group(group) => { + if let Expr::Lit(ExprLit { + lit: Lit::Str(lit_str), + .. + }) = *group.expr + { + (lit_str, None) + } else { + panic!("Expected a tuple with directory path and optional parameters, or a string literal for the directory path."); + } + }, + _ => panic!( + "Expected a tuple with directory path and optional parameters, or a string literal for the directory path." + ), + }; + + // Parse substitutions and pass to migration expander + let substitutions = parse_substitutions(parameters); + match migrate::expand_migrator_from_lit_dir(dir, substitutions) { Ok(ts) => ts.into(), Err(e) => { if let Some(parse_err) = e.downcast_ref::() { diff --git a/src/macros/mod.rs b/src/macros/mod.rs index 7f8ff747f9..81e35787e0 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -809,11 +809,19 @@ macro_rules! query_file_scalar_unchecked ( #[cfg(feature = "migrate")] #[macro_export] macro_rules! migrate { - ($dir:literal) => {{ - $crate::sqlx_macros::migrate!($dir) - }}; - - () => {{ + ($directory:literal, parameters = $parameters:expr) => { + $crate::sqlx_macros::migrate!(($directory, $parameters)); + }; + // Match when only parameters are provided + (parameters = $parameters:expr) => { + $crate::sqlx_macros::migrate!(("./migrations", $parameters)); + }; + // Match when only the directory is provided + ($dir:literal) => { + $crate::sqlx_macros::migrate!($dir); + }; + // Match when no arguments are provided + () => { $crate::sqlx_macros::migrate!("./migrations") - }}; + }; } diff --git a/tests/ui-tests.rs b/tests/ui-tests.rs index 4a5ca240e1..035c7be02a 100644 --- a/tests/ui-tests.rs +++ b/tests/ui-tests.rs @@ -44,3 +44,10 @@ fn ui_tests() { t.compile_fail("tests/ui/*.rs"); } + +#[test] +fn ui_migrate_tests() { + let t = trybuild::TestCases::new(); + t.compile_fail("tests/ui/migrate/invalid_key.rs"); + t.compile_fail("tests/ui/migrate/missing_parameter.rs"); +} diff --git a/tests/ui/migrate/invalid_key.rs b/tests/ui/migrate/invalid_key.rs new file mode 100644 index 0000000000..3edfaa8aa0 --- /dev/null +++ b/tests/ui/migrate/invalid_key.rs @@ -0,0 +1,4 @@ +fn main() { + //Fails due to invalid key + sqlx::migrate!("foo", parameters = [(123, "foo")]); +} diff --git a/tests/ui/migrate/invalid_key.stderr b/tests/ui/migrate/invalid_key.stderr new file mode 100644 index 0000000000..f92b52d11b --- /dev/null +++ b/tests/ui/migrate/invalid_key.stderr @@ -0,0 +1,8 @@ +error: proc macro panicked + --> tests/ui/migrate/invalid_key.rs:3:5 + | +3 | sqlx::migrate!("foo", parameters = [(123, "foo")]); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = help: message: Expected a string literal or a variable in key + = note: this error originates in the macro `sqlx::migrate` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/ui/migrate/migrations/20250423195520_create_users.down.sql b/tests/ui/migrate/migrations/20250423195520_create_users.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/tests/ui/migrate/migrations/20250423195520_create_users.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/tests/ui/migrate/migrations/20250423195520_create_users.up.sql b/tests/ui/migrate/migrations/20250423195520_create_users.up.sql new file mode 100644 index 0000000000..fc82f2cb9c --- /dev/null +++ b/tests/ui/migrate/migrations/20250423195520_create_users.up.sql @@ -0,0 +1,4 @@ +-- Add up migration script here +-- enable-substitution +CREATE USER ${my_user} WITH ENCRYPTED PASSWORD '${my_password}' INHERIT; +-- disable-substitution diff --git a/tests/ui/migrate/missing_parameter.rs b/tests/ui/migrate/missing_parameter.rs new file mode 100644 index 0000000000..a440b829ce --- /dev/null +++ b/tests/ui/migrate/missing_parameter.rs @@ -0,0 +1,5 @@ +fn main() { + //Fails due to missing migration parameter + let _shaggy = "shaggy"; + sqlx::migrate!("../../../../tests/ui/migrate/migrations", parameters = [("my_user", "scooby"), ("fooby", _shaggy)]); +} diff --git a/tests/ui/migrate/missing_parameter.stderr b/tests/ui/migrate/missing_parameter.stderr new file mode 100644 index 0000000000..63562022c3 --- /dev/null +++ b/tests/ui/migrate/missing_parameter.stderr @@ -0,0 +1,8 @@ +error: proc macro panicked + --> tests/ui/migrate/missing_parameter.rs:4:5 + | +4 | sqlx::migrate!("../../../../tests/ui/migrate/migrations", parameters = [("my_user", "scooby"), ("fooby", _shaggy)]); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = help: message: Missing substitution parameter: NoSuchVariable(NoSuchVariable { position: 50, name: "my_password" }) + = note: this error originates in the macro `sqlx::migrate` (in Nightly builds, run with -Z macro-backtrace for more info) From b2e293e8040ab9db33537167ddf610943a3d487d Mon Sep 17 00:00:00 2001 From: Keith Lohnes Date: Thu, 24 Apr 2025 13:28:19 -0400 Subject: [PATCH 05/19] Small refactors and formatting --- sqlx-cli/src/database.rs | 11 ++++++++- sqlx-cli/src/migrate.rs | 1 - sqlx-core/src/migrate/error.rs | 7 ++++-- sqlx-core/src/migrate/migration.rs | 27 ++++++++++++++++++++--- sqlx-core/src/migrate/migrator.rs | 13 +++++++---- sqlx-macros-core/src/migrate.rs | 17 ++------------ sqlx-macros/src/lib.rs | 25 +++++++++++---------- src/macros/mod.rs | 3 --- tests/ui/migrate/missing_parameter.stderr | 2 +- 9 files changed, 64 insertions(+), 42 deletions(-) diff --git a/sqlx-cli/src/database.rs b/sqlx-cli/src/database.rs index 171721b776..791a6b0164 100644 --- a/sqlx-cli/src/database.rs +++ b/sqlx-cli/src/database.rs @@ -57,7 +57,16 @@ pub async fn reset( pub async fn setup(migration_source: &str, connect_opts: &ConnectOpts) -> anyhow::Result<()> { create(connect_opts).await?; - migrate::run(migration_source, connect_opts, false, false, None, false, Vec::with_capacity(0)).await + migrate::run( + migration_source, + connect_opts, + false, + false, + None, + false, + Vec::with_capacity(0), + ) + .await } async fn ask_to_continue_drop(db_url: String) -> bool { diff --git a/sqlx-cli/src/migrate.rs b/sqlx-cli/src/migrate.rs index d15a9b437e..623720b18b 100644 --- a/sqlx-cli/src/migrate.rs +++ b/sqlx-cli/src/migrate.rs @@ -280,7 +280,6 @@ pub async fn run( params_from_env: bool, parameters: Vec<(String, String)>, ) -> anyhow::Result<()> { - println!("{:#?}", parameters); let migrator = Migrator::new(Path::new(migration_source)).await?; if let Some(target_version) = target_version { diff --git a/sqlx-core/src/migrate/error.rs b/sqlx-core/src/migrate/error.rs index 68c0895321..829f2bd2ed 100644 --- a/sqlx-core/src/migrate/error.rs +++ b/sqlx-core/src/migrate/error.rs @@ -40,6 +40,9 @@ pub enum MigrateError { )] Dirty(i64), - #[error("migration {0} was missing a parameter")] - MissingParameter(String), + #[error("migration {0} was missing a parameter '{1}' at line {2}, column {3}")] + MissingParameter(String, String, usize, usize), + + #[error("Invalid parameter syntax {0}")] + InvalidParameterSyntax(String), } diff --git a/sqlx-core/src/migrate/migration.rs b/sqlx-core/src/migrate/migration.rs index c9838e0e42..9a69b9bfd0 100644 --- a/sqlx-core/src/migrate/migration.rs +++ b/sqlx-core/src/migrate/migration.rs @@ -37,6 +37,21 @@ impl Migration { } } + fn name(&self) -> String { + let description = self.description.replace(' ', "_"); + match self.migration_type { + MigrationType::Simple => { + format!("{}_{}", self.version, description) + } + MigrationType::ReversibleUp => { + format!("{}_{}.{}", self.version, description, "up") + } + MigrationType::ReversibleDown => { + format!("{}_{}.{}", self.version, description, "down") + } + } + } + pub fn process_parameters( &self, params: &HashMap, @@ -69,8 +84,13 @@ impl Migration { } if substitution_enabled { - let substituted_line = subst::substitute(line, params) - .map_err(|e| MigrateError::MissingParameter(e.to_string()))?; + let substituted_line = subst::substitute(line, params).map_err(|e| match e { + subst::Error::NoSuchVariable(subst::error::NoSuchVariable { + position, + name, + }) => MigrateError::MissingParameter(self.name(), name, i + 1, position), + _ => MigrateError::InvalidParameterSyntax(e.to_string()), + })?; new_sql.push_str(&substituted_line); } else { new_sql.push_str(line); @@ -191,7 +211,8 @@ mod test { } #[test] - fn test_migration_process_parameters_missing_key_with_default_value() -> Result<(), MigrateError> { + fn test_migration_process_parameters_missing_key_with_default_value() -> Result<(), MigrateError> + { const CREATE_TABLE: &str = r#" -- enable-substitution CREATE TABLE foo ( diff --git a/sqlx-core/src/migrate/migrator.rs b/sqlx-core/src/migrate/migrator.rs index 75c1820758..5627972a33 100644 --- a/sqlx-core/src/migrate/migrator.rs +++ b/sqlx-core/src/migrate/migrator.rs @@ -74,7 +74,10 @@ impl Migrator { K: Into, V: Into, { - let map: HashMap = params.into_iter().map(|(k, v)| (k.into(), v.into())).collect(); + let map: HashMap = params + .into_iter() + .map(|(k, v)| (k.into(), v.into())) + .collect(); self.template_params = Some(map); self } @@ -113,8 +116,11 @@ impl Migrator { self } - /// Specify whether template parameters for migrations should be read from the environment - pub fn set_template_parameters_from_env(&mut self, template_paramaters_from_env: bool) -> &Self { + /// Specify whether template parameters for migrations should be read from the environment + pub fn set_template_parameters_from_env( + &mut self, + template_paramaters_from_env: bool, + ) -> &Self { self.template_parameters_from_env = template_paramaters_from_env; self } @@ -197,7 +203,6 @@ impl Migrator { .map(|m| (m.version, m)) .collect(); - let env_params = if self.template_parameters_from_env { Some(std::env::vars().collect()) } else { diff --git a/sqlx-macros-core/src/migrate.rs b/sqlx-macros-core/src/migrate.rs index ba57499dc0..38cd481cf2 100644 --- a/sqlx-macros-core/src/migrate.rs +++ b/sqlx-macros-core/src/migrate.rs @@ -102,8 +102,6 @@ pub(crate) fn expand_migrator( path: &Path, parameters: Option>, ) -> crate::Result { - const ENABLE_SUBSTITUTION: &str = "-- enable-substitution"; - const DISABLE_SUBSTITUTION: &str = "-- disable-substitution"; let path = path.canonicalize().map_err(|e| { format!( "error canonicalizing migration directory {}: {e}", @@ -112,23 +110,12 @@ pub(crate) fn expand_migrator( })?; // Use the same code path to resolve migrations at compile time and runtime. - let mut substitution_enabled = false; let migrations = sqlx_core::migrate::resolve_blocking(&path)? .into_iter() .map(|(migration, path)| { if let Some(ref params) = parameters { - for line in migration.sql.lines() { - let trimmed_line = line.trim(); - if trimmed_line == ENABLE_SUBSTITUTION { - substitution_enabled = true; - continue; - } else if trimmed_line == DISABLE_SUBSTITUTION { - substitution_enabled = false; - continue; - } - if substitution_enabled { - subst::substitute(line, params).expect("Missing substitution parameter"); - } + if let Err(e) = migration.process_parameters(params) { + panic!("Error processing parameters: {e}"); } } QuoteMigration { migration, path } diff --git a/sqlx-macros/src/lib.rs b/sqlx-macros/src/lib.rs index 8b5bd1ff82..ef8863b90d 100644 --- a/sqlx-macros/src/lib.rs +++ b/sqlx-macros/src/lib.rs @@ -71,20 +71,21 @@ pub fn migrate(input: TokenStream) -> TokenStream { // Extract directory path, handling both direct literals and grouped literals fn extract_dir(expr: Option) -> LitStr { - if let Some(Expr::Lit(ExprLit { - lit: Lit::Str(lit_str), - .. - })) = expr - { - return lit_str; - } else if let Some(Expr::Group(group)) = expr { - if let Expr::Lit(ExprLit { - lit: Lit::Str(lit_str), + match expr { + Some(Expr::Lit(ExprLit { + lit: Lit::Str(literal), .. - }) = *group.expr - { - return lit_str; + })) => return literal, + Some(Expr::Group(group)) => { + if let Expr::Lit(ExprLit { + lit: Lit::Str(literal), + .. + }) = *group.expr + { + return literal; + } } + _ => {} } panic!("Expected a string literal for the directory path."); } diff --git a/src/macros/mod.rs b/src/macros/mod.rs index 81e35787e0..aecc60c957 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -812,15 +812,12 @@ macro_rules! migrate { ($directory:literal, parameters = $parameters:expr) => { $crate::sqlx_macros::migrate!(($directory, $parameters)); }; - // Match when only parameters are provided (parameters = $parameters:expr) => { $crate::sqlx_macros::migrate!(("./migrations", $parameters)); }; - // Match when only the directory is provided ($dir:literal) => { $crate::sqlx_macros::migrate!($dir); }; - // Match when no arguments are provided () => { $crate::sqlx_macros::migrate!("./migrations") }; diff --git a/tests/ui/migrate/missing_parameter.stderr b/tests/ui/migrate/missing_parameter.stderr index 63562022c3..0717bc64c4 100644 --- a/tests/ui/migrate/missing_parameter.stderr +++ b/tests/ui/migrate/missing_parameter.stderr @@ -4,5 +4,5 @@ error: proc macro panicked 4 | sqlx::migrate!("../../../../tests/ui/migrate/migrations", parameters = [("my_user", "scooby"), ("fooby", _shaggy)]); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | - = help: message: Missing substitution parameter: NoSuchVariable(NoSuchVariable { position: 50, name: "my_password" }) + = help: message: Error processing parameters: migration 20250423195520_create_users.up was missing a parameter 'my_password' at line 3, column 50 = note: this error originates in the macro `sqlx::migrate` (in Nightly builds, run with -Z macro-backtrace for more info) From f09db3a23ed6610340a88fe654b7c0a980d18e94 Mon Sep 17 00:00:00 2001 From: Keith Lohnes Date: Mon, 28 Apr 2025 10:43:35 -0400 Subject: [PATCH 06/19] Remove println --- sqlx-cli/src/migrate.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/sqlx-cli/src/migrate.rs b/sqlx-cli/src/migrate.rs index 623720b18b..e05f8826e7 100644 --- a/sqlx-cli/src/migrate.rs +++ b/sqlx-cli/src/migrate.rs @@ -280,7 +280,6 @@ pub async fn run( params_from_env: bool, parameters: Vec<(String, String)>, ) -> anyhow::Result<()> { - println!("{:#?}", parameters); let migrator = Migrator::new(Path::new(migration_source)).await?; if let Some(target_version) = target_version { if !migrator.version_exists(target_version) { From 396dec54a2153c1e528d3bbc9cbf736f277be144 Mon Sep 17 00:00:00 2001 From: Keith Lohnes Date: Mon, 28 Apr 2025 11:05:39 -0400 Subject: [PATCH 07/19] Add some documentation; Implement subst for revert in cli --- README.md | 20 ++++++++++++++++++++ sqlx-cli/README.md | 22 ++++++++++++++++++++++ sqlx-cli/src/lib.rs | 8 ++++++-- sqlx-cli/src/migrate.rs | 16 ++++++++++++++++ sqlx-cli/src/opt.rs | 13 +++++++++++-- 5 files changed, 75 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index cc0ecf2e66..94a6314043 100644 --- a/README.md +++ b/README.md @@ -458,6 +458,26 @@ opt-level = 3 1 The `dotenv` crate itself appears abandoned as of [December 2021](https://github.com/dotenv-rs/dotenv/issues/74) so we now use the `dotenvy` crate instead. The file format is the same. +## Parameter Substitution for Migrations + +You can parameterize migrations using parameters, either from the environment or passed in from the cli or to the Migrator. + +For example: + +```sql +-- enable-substitution +CREATE USER ${USER_FROM_ENV} WITH PASSWORD ${PASSWORD_FROM_ENV} +-- disable-substituion +``` + +We use the [subst](https://crates.io/crates/subst) to support substitution. sqlx supports + +- Short format: `$NAME` +- Long format: `${NAME}` +- Default values: `${NAME:Bob}` +- Recursive Substitution in Default Values: `${NAME: Bob ${OTHER_NAME: and Alice}}` + + ## Safety This crate uses `#![forbid(unsafe_code)]` to ensure everything is implemented in 100% Safe Rust. diff --git a/sqlx-cli/README.md b/sqlx-cli/README.md index b20461b8fd..eabf68c101 100644 --- a/sqlx-cli/README.md +++ b/sqlx-cli/README.md @@ -65,6 +65,18 @@ any scripts that are still pending. --- +Users can also provide parameters through environment variables or pass them in manually. + +```bash +sqlx migrate run --params-from-env +``` + +```bash +sqlx migrate run --params key:value,key1,value1 +``` + +--- + Users can provide the directory for the migration scripts to `sqlx migrate` subcommands with the `--source` flag. ```bash @@ -105,6 +117,16 @@ Creating migrations/20211001154420_.up.sql Creating migrations/20211001154420_.down.sql ``` +Users can also provide parameters through environment variables or pass them in manually, just as they did with the run command. + +```bash +sqlx migrate revert --params-from-env +``` + +```bash +sqlx migrate revert --params key:value,key1,value1 +``` + ### Enable building in "offline mode" with `query!()` There are 2 steps to building with "offline mode": diff --git a/sqlx-cli/src/lib.rs b/sqlx-cli/src/lib.rs index dee8c948dc..7906796eb6 100644 --- a/sqlx-cli/src/lib.rs +++ b/sqlx-cli/src/lib.rs @@ -67,7 +67,7 @@ async fn do_run(opt: Opt) -> Result<()> { connect_opts, target_version, params_from_env, - parameters, + params, } => { migrate::run( &source, @@ -76,7 +76,7 @@ async fn do_run(opt: Opt) -> Result<()> { *ignore_missing, target_version, params_from_env, - parameters, + params, ) .await? } @@ -86,6 +86,8 @@ async fn do_run(opt: Opt) -> Result<()> { ignore_missing, connect_opts, target_version, + params_from_env, + params, } => { migrate::revert( &source, @@ -93,6 +95,8 @@ async fn do_run(opt: Opt) -> Result<()> { dry_run, *ignore_missing, target_version, + params_from_env, + params, ) .await? } diff --git a/sqlx-cli/src/migrate.rs b/sqlx-cli/src/migrate.rs index e05f8826e7..70e83ad3b0 100644 --- a/sqlx-cli/src/migrate.rs +++ b/sqlx-cli/src/migrate.rs @@ -385,6 +385,8 @@ pub async fn revert( dry_run: bool, ignore_missing: bool, target_version: Option, + params_from_env: bool, + parameters: Vec<(String, String)>, ) -> anyhow::Result<()> { let migrator = Migrator::new(Path::new(migration_source)).await?; if let Some(target_version) = target_version { @@ -422,6 +424,15 @@ pub async fn revert( .collect(); let mut is_applied = false; + + let env_params: HashMap<_, _> = if params_from_env { + std::env::vars().collect() + } else { + HashMap::with_capacity(0) + }; + + let params: HashMap<_, _> = parameters.into_iter().collect(); + for migration in migrator.iter().rev() { if !migration.migration_type.is_down_migration() { // Skipping non down migration @@ -436,6 +447,11 @@ pub async fn revert( let elapsed = if dry_run || skip { Duration::new(0, 0) } else { + if params_from_env { + migration.process_parameters(&env_params)?; + } else if !params.is_empty() { + migration.process_parameters(¶ms)?; + } conn.revert(migration).await? }; let text = if skip { diff --git a/sqlx-cli/src/opt.rs b/sqlx-cli/src/opt.rs index d2f8030355..30ed3da26b 100644 --- a/sqlx-cli/src/opt.rs +++ b/sqlx-cli/src/opt.rs @@ -188,9 +188,9 @@ pub enum MigrateCommand { params_from_env: bool, #[clap(long, short, value_parser = parse_key_val::, num_args = 1, value_delimiter=',')] - /// Provide template parameters for substitution in migrations, e.g. --parameters + /// Provide template parameters for substitution in migrations, e.g. --params /// key:value,key2:value2 - parameters: Vec<(String, String)>, + params: Vec<(String, String)>, }, /// Revert the latest migration with a down file. @@ -213,6 +213,15 @@ pub enum MigrateCommand { /// at the target version, then no-op. #[clap(long)] target_version: Option, + + #[clap(long)] + /// Template parameters for substitution in migrations from environment variables + params_from_env: bool, + + #[clap(long, short, value_parser = parse_key_val::, num_args = 1, value_delimiter=',')] + /// Provide template parameters for substitution in migrations, e.g. --params + /// key:value,key2:value2 + params: Vec<(String, String)>, }, /// List all available migrations. From 9c88f326e9859cb3acd4f51fa1eed49eae24b33f Mon Sep 17 00:00:00 2001 From: Keith Lohnes Date: Mon, 28 Apr 2025 11:10:13 -0400 Subject: [PATCH 08/19] Add parameters for undo in migrator --- sqlx-core/src/migrate/migrator.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sqlx-core/src/migrate/migrator.rs b/sqlx-core/src/migrate/migrator.rs index 5627972a33..4bd5f0c344 100644 --- a/sqlx-core/src/migrate/migrator.rs +++ b/sqlx-core/src/migrate/migrator.rs @@ -286,6 +286,12 @@ impl Migrator { .map(|m| (m.version, m)) .collect(); + let env_params = if self.template_parameters_from_env { + Some(std::env::vars().collect()) + } else { + None + }; + for migration in self .iter() .rev() @@ -293,6 +299,11 @@ impl Migrator { .filter(|m| applied_migrations.contains_key(&m.version)) .filter(|m| m.version > target) { + if self.template_parameters_from_env { + migration.process_parameters(env_params.as_ref().unwrap())?; + } else if let Some(params) = self.template_params.as_ref() { + migration.process_parameters(params)?; + } conn.revert(migration).await?; } From e686352199dae98fafac746d528a9ae0b9ab1e31 Mon Sep 17 00:00:00 2001 From: Keith Lohnes Date: Mon, 28 Apr 2025 11:50:28 -0400 Subject: [PATCH 09/19] Fix match statemnt in migration test --- sqlx-core/src/migrate/migration.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlx-core/src/migrate/migration.rs b/sqlx-core/src/migrate/migration.rs index 9a69b9bfd0..b0173a153f 100644 --- a/sqlx-core/src/migrate/migration.rs +++ b/sqlx-core/src/migrate/migration.rs @@ -202,7 +202,7 @@ mod test { Cow::Owned(CREATE_TABLE.to_string()), true, ); - let Err(MigrateError::MissingParameter(_)) = + let Err(MigrateError::MissingParameter(..)) = migration.process_parameters(&HashMap::with_capacity(0)) else { panic!("Missing env var not caught in process parameters missing env var") From 40a7c4e89e0681da9b7dafa3dda034fc5f8dd7b0 Mon Sep 17 00:00:00 2001 From: Keith Lohnes Date: Mon, 28 Apr 2025 12:47:48 -0400 Subject: [PATCH 10/19] Fix default migrations directory for macro --- sqlx-macros/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sqlx-macros/src/lib.rs b/sqlx-macros/src/lib.rs index ef8863b90d..848b60779a 100644 --- a/sqlx-macros/src/lib.rs +++ b/sqlx-macros/src/lib.rs @@ -139,6 +139,12 @@ pub fn migrate(input: TokenStream) -> TokenStream { let mut elems = elems.into_iter(); (extract_dir(elems.next()), elems.next()) } + Expr::Lit(ExprLit { + lit: Lit::Str(lit_str), + .. + }) => { + (lit_str, None) + } Expr::Group(group) => { if let Expr::Lit(ExprLit { lit: Lit::Str(lit_str), From 650f911fe8b1c7e169e2aa8be8f2d4439d0fd345 Mon Sep 17 00:00:00 2001 From: Keith Lohnes Date: Mon, 28 Apr 2025 13:32:50 -0400 Subject: [PATCH 11/19] Fix doc test failure --- src/macros/test.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/macros/test.md b/src/macros/test.md index 30de8070f6..1a3e1c581d 100644 --- a/src/macros/test.md +++ b/src/macros/test.md @@ -133,7 +133,9 @@ use sqlx::{PgPool, Row}; # migrations: Cow::Borrowed(&[]), # ignore_missing: false, # locking: true, -# no_tx: false +# no_tx: false, +# template_params: None, +# template_parameters_from_env: false, # }; # } From e35498d64865f40e3542aa3f1b5ba304566e56e7 Mon Sep 17 00:00:00 2001 From: Keith Lohnes Date: Mon, 28 Apr 2025 14:04:46 -0400 Subject: [PATCH 12/19] Put ui test behind migrate flag --- tests/ui-tests.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/ui-tests.rs b/tests/ui-tests.rs index 035c7be02a..fbdd14aa6f 100644 --- a/tests/ui-tests.rs +++ b/tests/ui-tests.rs @@ -47,7 +47,9 @@ fn ui_tests() { #[test] fn ui_migrate_tests() { - let t = trybuild::TestCases::new(); - t.compile_fail("tests/ui/migrate/invalid_key.rs"); - t.compile_fail("tests/ui/migrate/missing_parameter.rs"); + if cfg!(feature = "migrate") { + let t = trybuild::TestCases::new(); + t.compile_fail("tests/ui/migrate/invalid_key.rs"); + t.compile_fail("tests/ui/migrate/missing_parameter.rs"); + } } From 44f9d1a926c6c27295630e42467df4674d90b566 Mon Sep 17 00:00:00 2001 From: Keith Lohnes Date: Mon, 28 Apr 2025 15:03:05 -0400 Subject: [PATCH 13/19] process_parameters does not mutate self Was treating it like it did, so it wasn't working --- sqlx-cli/src/migrate.rs | 21 +++++++++++---------- sqlx-core/src/migrate/migrator.rs | 16 ++++++++++------ 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/sqlx-cli/src/migrate.rs b/sqlx-cli/src/migrate.rs index 70e83ad3b0..adb5f87f37 100644 --- a/sqlx-cli/src/migrate.rs +++ b/sqlx-cli/src/migrate.rs @@ -280,6 +280,7 @@ pub async fn run( params_from_env: bool, parameters: Vec<(String, String)>, ) -> anyhow::Result<()> { + println!("parameters {:#?}", parameters); let migrator = Migrator::new(Path::new(migration_source)).await?; if let Some(target_version) = target_version { if !migrator.version_exists(target_version) { @@ -341,12 +342,12 @@ pub async fn run( let elapsed = if dry_run || skip { Duration::new(0, 0) + } else if params_from_env { + conn.apply(&migration.process_parameters(&env_params)?) + .await? + } else if !params.is_empty() { + conn.apply(&migration.process_parameters(¶ms)?).await? } else { - if params_from_env { - migration.process_parameters(&env_params)?; - } else if !params.is_empty() { - migration.process_parameters(¶ms)?; - } conn.apply(migration).await? }; let text = if skip { @@ -446,12 +447,12 @@ pub async fn revert( let elapsed = if dry_run || skip { Duration::new(0, 0) + } else if params_from_env { + conn.revert(&migration.process_parameters(&env_params)?) + .await? + } else if !params.is_empty() { + conn.revert(&migration.process_parameters(¶ms)?).await? } else { - if params_from_env { - migration.process_parameters(&env_params)?; - } else if !params.is_empty() { - migration.process_parameters(¶ms)?; - } conn.revert(migration).await? }; let text = if skip { diff --git a/sqlx-core/src/migrate/migrator.rs b/sqlx-core/src/migrate/migrator.rs index 4bd5f0c344..6100508350 100644 --- a/sqlx-core/src/migrate/migrator.rs +++ b/sqlx-core/src/migrate/migrator.rs @@ -222,11 +222,13 @@ impl Migrator { } None => { if self.template_parameters_from_env { - migration.process_parameters(env_params.as_ref().unwrap())?; + conn.apply(&migration.process_parameters(env_params.as_ref().unwrap())?) + .await?; } else if let Some(params) = self.template_params.as_ref() { - migration.process_parameters(params)?; + conn.apply(&migration.process_parameters(params)?).await?; + } else { + conn.apply(migration).await?; } - conn.apply(migration).await?; } } } @@ -300,11 +302,13 @@ impl Migrator { .filter(|m| m.version > target) { if self.template_parameters_from_env { - migration.process_parameters(env_params.as_ref().unwrap())?; + conn.revert(&migration.process_parameters(env_params.as_ref().unwrap())?) + .await?; } else if let Some(params) = self.template_params.as_ref() { - migration.process_parameters(params)?; + conn.revert(&migration.process_parameters(params)?).await?; + } else { + conn.revert(migration).await?; } - conn.revert(migration).await?; } // unlock the migrator to allow other migrators to run From 1f8fdcbd546a0310de186dafff62c8cef68fbda9 Mon Sep 17 00:00:00 2001 From: Keith Lohnes Date: Mon, 28 Apr 2025 15:04:47 -0400 Subject: [PATCH 14/19] Remove println --- sqlx-cli/src/migrate.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/sqlx-cli/src/migrate.rs b/sqlx-cli/src/migrate.rs index adb5f87f37..50d74f1464 100644 --- a/sqlx-cli/src/migrate.rs +++ b/sqlx-cli/src/migrate.rs @@ -280,7 +280,6 @@ pub async fn run( params_from_env: bool, parameters: Vec<(String, String)>, ) -> anyhow::Result<()> { - println!("parameters {:#?}", parameters); let migrator = Migrator::new(Path::new(migration_source)).await?; if let Some(target_version) = target_version { if !migrator.version_exists(target_version) { From 21c75bad9a47977f8040879a630eaea192e8cdc9 Mon Sep 17 00:00:00 2001 From: Keith Lohnes Date: Wed, 6 Aug 2025 09:50:49 -0400 Subject: [PATCH 15/19] Restore migration parameter functionality and fix tests - Restored parameter support in migrate macro while maintaining compatibility with main branch API - Fixed test code to use SqlStr instead of Cow for SQL migration content - All parameter processing tests now pass: - test_migration_process_parameters_with_substitution - test_migration_process_parameters_no_substitution - test_migration_process_parameters_missing_key - test_migration_process_parameters_missing_key_with_default_value - UI tests for parameter validation also pass - Migration functionality works with both simple path and parameter-based approaches --- sqlx-core/src/migrate/migration.rs | 16 ++-- sqlx-macros/src/lib.rs | 118 ++++++++++++++++++++++++++++- src/macros/mod.rs | 7 +- 3 files changed, 129 insertions(+), 12 deletions(-) diff --git a/sqlx-core/src/migrate/migration.rs b/sqlx-core/src/migrate/migration.rs index 097c0e1e09..64f0da2f04 100644 --- a/sqlx-core/src/migrate/migration.rs +++ b/sqlx-core/src/migrate/migration.rs @@ -2,7 +2,7 @@ use sha2::{Digest, Sha384}; use std::borrow::Cow; use std::collections::HashMap; -use crate::sql_str::SqlStr; +use crate::sql_str::{SqlStr, SqlSafeStr}; use super::{MigrateError, MigrationType}; @@ -85,10 +85,10 @@ impl Migration { no_tx, } = self; - let mut new_sql = String::with_capacity(sql.len()); + let mut new_sql = String::with_capacity(sql.as_str().len()); let mut substitution_enabled = false; - for (i, line) in sql.lines().enumerate() { + for (i, line) in sql.as_str().lines().enumerate() { if i != 0 { new_sql.push('\n') } @@ -121,7 +121,7 @@ impl Migration { version: *version, description: description.clone(), migration_type: *migration_type, - sql: Cow::Owned(new_sql), + sql: crate::sql_str::AssertSqlSafe(new_sql).into_sql_str(), checksum: checksum.clone(), no_tx: *no_tx, }) @@ -183,7 +183,7 @@ mod test { 1, Cow::Owned("test a simple parameter substitution".to_string()), crate::migrate::MigrationType::Simple, - Cow::Owned(CREATE_USER.to_string()), + crate::sql_str::AssertSqlSafe(CREATE_USER.to_string()).into_sql_str(), true, ); let result = migration.process_parameters(&HashMap::from([( @@ -206,7 +206,7 @@ mod test { 1, std::borrow::Cow::Owned("test a simple parameter substitution".to_string()), crate::migrate::MigrationType::Simple, - Cow::Owned(CREATE_TABLE.to_string()), + crate::sql_str::AssertSqlSafe(CREATE_TABLE.to_string()).into_sql_str(), true, ); let result = migration.process_parameters(&HashMap::from([( @@ -233,7 +233,7 @@ mod test { 1, Cow::Owned("test a simple parameter substitution".to_string()), crate::migrate::MigrationType::Simple, - Cow::Owned(CREATE_TABLE.to_string()), + crate::sql_str::AssertSqlSafe(CREATE_TABLE.to_string()).into_sql_str(), true, ); let Err(MigrateError::MissingParameter(..)) = @@ -269,7 +269,7 @@ mod test { 1, Cow::Owned("test a simple parameter substitution".to_string()), crate::migrate::MigrationType::Simple, - Cow::Owned(CREATE_TABLE.to_string()), + crate::sql_str::AssertSqlSafe(CREATE_TABLE.to_string()).into_sql_str(), true, ); let result = migration.process_parameters(&HashMap::with_capacity(0))?; diff --git a/sqlx-macros/src/lib.rs b/sqlx-macros/src/lib.rs index 13e5e54754..b938366b5e 100644 --- a/sqlx-macros/src/lib.rs +++ b/sqlx-macros/src/lib.rs @@ -66,10 +66,122 @@ pub fn derive_from_row(input: TokenStream) -> TokenStream { #[cfg(feature = "migrate")] #[proc_macro] pub fn migrate(input: TokenStream) -> TokenStream { - use syn::{parse_macro_input, LitStr}; + use std::collections::HashMap; + use syn::{parse_macro_input, Expr, ExprArray, ExprLit, ExprPath, ExprTuple, Lit, LitStr}; + use quote::quote; - let input = syn::parse_macro_input!(input as Option); - match migrate::expand(input) { + // Extract directory path, handling both direct literals and grouped literals + fn extract_dir(expr: Option) -> LitStr { + match expr { + Some(Expr::Lit(ExprLit { + lit: Lit::Str(literal), + .. + })) => return literal, + Some(Expr::Group(group)) => { + if let Expr::Lit(ExprLit { + lit: Lit::Str(literal), + .. + }) = *group.expr + { + return literal; + } + } + _ => {} + } + panic!("Expected a string literal for the directory path."); + } + + // Extract a `String` value from an expression (either a string literal or a variable) + fn extract_value(expr: Expr, location: &str) -> String { + match expr { + Expr::Lit(ExprLit { + lit: Lit::Str(lit_str), + .. + }) => lit_str.value(), + Expr::Path(ExprPath { path, .. }) => path.segments.last().unwrap().ident.to_string(), + _ => panic!("Expected a string literal or a variable in {location}"), + } + } + + // Parse substitutions, expecting an array of tuples (String, Expr) + fn parse_substitutions(expr: Option) -> Option> { + let Expr::Group(group) = expr? else { + return None; + }; + let Expr::Array(ExprArray { elems, .. }) = *group.expr else { + panic!("Expected an array of tuples (String, Expr)."); + }; + + let mut map = HashMap::new(); + for elem in elems { + let Expr::Tuple(ExprTuple { + elems: tuple_elems, .. + }) = elem + else { + panic!("Expected a tuple (String, Expr). Got {:#?}", elem); + }; + + let mut tuple_elems = tuple_elems.into_iter(); + + let key = extract_value(tuple_elems.next().expect("Missing key in tuple."), "key"); + let value = extract_value( + tuple_elems.next().expect("Missing value in tuple."), + "value", + ); + map.insert(key, value); + } + Some(map) + } + + // Handle both the simple case (just path) and the tuple case (path + parameters) + let input_result: std::result::Result, syn::Error> = syn::parse(input.clone()); + if let Ok(simple_input) = input_result { + // Simple case: just a path or no arguments + return match migrate::expand(simple_input) { + Ok(ts) => ts.into(), + Err(e) => { + if let Some(parse_err) = e.downcast_ref::() { + parse_err.to_compile_error().into() + } else { + let msg = e.to_string(); + quote!(::std::compile_error!(#msg)).into() + } + } + }; + } + + // Complex case: parse tuple with parameters + let exp = parse_macro_input!(input as syn::Expr); + let (dir, parameters) = match exp { + Expr::Tuple(ExprTuple { elems, .. }) => { + let mut elems = elems.into_iter(); + (extract_dir(elems.next()), elems.next()) + } + Expr::Lit(ExprLit { + lit: Lit::Str(lit_str), + .. + }) => { + (lit_str, None) + } + Expr::Group(group) => { + if let Expr::Lit(ExprLit { + lit: Lit::Str(lit_str), + .. + }) = *group.expr + { + (lit_str, None) + } else { + panic!("Expected a tuple with directory path and optional parameters, or a string literal for the directory path."); + } + }, + _ => panic!( + "Expected a tuple with directory path and optional parameters, or a string literal for the directory path." + ), + }; + + // Parse substitutions and pass to migration expander + let substitutions = parse_substitutions(parameters); + match migrate::expand_migrator_from_lit_dir(dir, substitutions) { Ok(ts) => ts.into(), Err(e) => { if let Some(parse_err) = e.downcast_ref::() { diff --git a/src/macros/mod.rs b/src/macros/mod.rs index 0db6f0c2e7..366ef738f7 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -861,10 +861,15 @@ macro_rules! query_file_scalar_unchecked ( #[cfg(feature = "migrate")] #[macro_export] macro_rules! migrate { + ($directory:literal, parameters = $parameters:expr) => { + $crate::sqlx_macros::migrate!(($directory, $parameters)) + }; + (parameters = $parameters:expr) => { + $crate::sqlx_macros::migrate!(("./migrations", $parameters)) + }; ($dir:literal) => {{ $crate::sqlx_macros::migrate!($dir) }}; - () => {{ $crate::sqlx_macros::migrate!() }}; From 2dd879febcf67839999a7a784b63e833c1cf20f7 Mon Sep 17 00:00:00 2001 From: Keith Lohnes Date: Wed, 6 Aug 2025 10:01:09 -0400 Subject: [PATCH 16/19] More test fixes --- sqlx-cli/src/database.rs | 12 +++++++++++- sqlx-cli/src/migrate.rs | 26 ++++++++++++++++++++------ sqlx-cli/src/opt.rs | 2 +- sqlx-core/src/migrate/migration.rs | 2 +- sqlx-core/src/migrate/migrator.rs | 20 ++++++++++++++------ sqlx-macros/src/lib.rs | 2 +- 6 files changed, 48 insertions(+), 16 deletions(-) diff --git a/sqlx-cli/src/database.rs b/sqlx-cli/src/database.rs index eaba46eed9..b0cde2defa 100644 --- a/sqlx-cli/src/database.rs +++ b/sqlx-cli/src/database.rs @@ -62,7 +62,17 @@ pub async fn setup( connect_opts: &ConnectOpts, ) -> anyhow::Result<()> { create(connect_opts).await?; - migrate::run(config, migration_source, connect_opts, false, false, None).await + migrate::run( + config, + migration_source, + connect_opts, + false, + false, + None, + false, + Vec::new(), + ) + .await } async fn ask_to_continue_drop(db_url: String) -> bool { diff --git a/sqlx-cli/src/migrate.rs b/sqlx-cli/src/migrate.rs index 354d28b0ed..b846ada18b 100644 --- a/sqlx-cli/src/migrate.rs +++ b/sqlx-cli/src/migrate.rs @@ -293,10 +293,17 @@ pub async fn run( let elapsed = if dry_run || skip { Duration::new(0, 0) } else if params_from_env { - conn.apply(&migration.process_parameters(&env_params)?) - .await? + conn.apply( + config.migrate.table_name(), + &migration.process_parameters(&env_params)?, + ) + .await? } else if !params.is_empty() { - conn.apply(&migration.process_parameters(¶ms)?).await? + conn.apply( + config.migrate.table_name(), + &migration.process_parameters(¶ms)?, + ) + .await? } else { conn.apply(config.migrate.table_name(), migration).await? }; @@ -408,10 +415,17 @@ pub async fn revert( let elapsed = if dry_run || skip { Duration::new(0, 0) } else if params_from_env { - conn.revert(&migration.process_parameters(&env_params)?) - .await? + conn.revert( + config.migrate.table_name(), + &migration.process_parameters(&env_params)?, + ) + .await? } else if !params.is_empty() { - conn.revert(&migration.process_parameters(¶ms)?).await? + conn.revert( + config.migrate.table_name(), + &migration.process_parameters(¶ms)?, + ) + .await? } else { conn.revert(config.migrate.table_name(), migration).await? }; diff --git a/sqlx-cli/src/opt.rs b/sqlx-cli/src/opt.rs index 86432d5af8..63fa7143ab 100644 --- a/sqlx-cli/src/opt.rs +++ b/sqlx-cli/src/opt.rs @@ -2,7 +2,6 @@ use crate::config::migrate::{DefaultMigrationType, DefaultVersioning}; use crate::config::Config; use anyhow::Context; use chrono::Utc; -use std::error::Error; use clap::{ builder::{styling::AnsiColor, Styles}, Args, Parser, @@ -11,6 +10,7 @@ use clap::{ use clap_complete::Shell; use sqlx::migrate::{MigrateError, Migrator, ResolveWith}; use std::env; +use std::error::Error; use std::ops::{Deref, Not}; use std::path::PathBuf; diff --git a/sqlx-core/src/migrate/migration.rs b/sqlx-core/src/migrate/migration.rs index 64f0da2f04..e0960892b0 100644 --- a/sqlx-core/src/migrate/migration.rs +++ b/sqlx-core/src/migrate/migration.rs @@ -2,7 +2,7 @@ use sha2::{Digest, Sha384}; use std::borrow::Cow; use std::collections::HashMap; -use crate::sql_str::{SqlStr, SqlSafeStr}; +use crate::sql_str::{SqlSafeStr, SqlStr}; use super::{MigrateError, MigrationType}; diff --git a/sqlx-core/src/migrate/migrator.rs b/sqlx-core/src/migrate/migrator.rs index d391e850e0..c0e8f13f11 100644 --- a/sqlx-core/src/migrate/migrator.rs +++ b/sqlx-core/src/migrate/migrator.rs @@ -259,10 +259,14 @@ impl Migrator { } None => { if self.template_parameters_from_env { - conn.apply(&self.table_name, &migration.process_parameters(env_params.as_ref().unwrap())?) - .await?; + conn.apply( + &self.table_name, + &migration.process_parameters(env_params.as_ref().unwrap())?, + ) + .await?; } else if let Some(params) = self.template_params.as_ref() { - conn.apply(&self.table_name, &migration.process_parameters(params)?).await?; + conn.apply(&self.table_name, &migration.process_parameters(params)?) + .await?; } else { conn.apply(&self.table_name, migration).await?; } @@ -339,10 +343,14 @@ impl Migrator { .filter(|m| m.version > target) { if self.template_parameters_from_env { - conn.revert(&self.table_name, &migration.process_parameters(env_params.as_ref().unwrap())?) - .await?; + conn.revert( + &self.table_name, + &migration.process_parameters(env_params.as_ref().unwrap())?, + ) + .await?; } else if let Some(params) = self.template_params.as_ref() { - conn.revert(&self.table_name, &migration.process_parameters(params)?).await?; + conn.revert(&self.table_name, &migration.process_parameters(params)?) + .await?; } else { conn.revert(&self.table_name, migration).await?; } diff --git a/sqlx-macros/src/lib.rs b/sqlx-macros/src/lib.rs index b938366b5e..7f85a06652 100644 --- a/sqlx-macros/src/lib.rs +++ b/sqlx-macros/src/lib.rs @@ -66,9 +66,9 @@ pub fn derive_from_row(input: TokenStream) -> TokenStream { #[cfg(feature = "migrate")] #[proc_macro] pub fn migrate(input: TokenStream) -> TokenStream { + use quote::quote; use std::collections::HashMap; use syn::{parse_macro_input, Expr, ExprArray, ExprLit, ExprPath, ExprTuple, Lit, LitStr}; - use quote::quote; // Extract directory path, handling both direct literals and grouped literals fn extract_dir(expr: Option) -> LitStr { From 2d96f40568d7c9a4b851ed92474dd7a80f8ddb98 Mon Sep 17 00:00:00 2001 From: Keith Lohnes Date: Sun, 10 Aug 2025 13:31:13 -0400 Subject: [PATCH 17/19] Fix sqllite describe --- benches/sqlite/describe.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/benches/sqlite/describe.rs b/benches/sqlite/describe.rs index 5d353b3d3c..585bf9a4f9 100644 --- a/benches/sqlite/describe.rs +++ b/benches/sqlite/describe.rs @@ -3,12 +3,13 @@ use criterion::Criterion; use criterion::{criterion_group, criterion_main}; use sqlx::sqlite::{Sqlite, SqliteConnection}; +use sqlx::SqlSafeStr; use sqlx::Executor; use sqlx_test::new; // Here we have an async function to benchmark async fn do_describe_trivial(db: &std::cell::RefCell) { - db.borrow_mut().describe("select 1").await.unwrap(); + db.borrow_mut().describe("select 1".into_sql_str()).await.unwrap(); } async fn do_describe_recursive(db: &std::cell::RefCell) { @@ -27,7 +28,7 @@ async fn do_describe_recursive(db: &std::cell::RefCell) { begin_date FROM schedule GROUP BY begin_date - "#, + "#.into_sql_str(), ) .await .unwrap(); @@ -35,14 +36,14 @@ async fn do_describe_recursive(db: &std::cell::RefCell) { async fn do_describe_insert(db: &std::cell::RefCell) { db.borrow_mut() - .describe("INSERT INTO tweet (id, text) VALUES (2, 'Hello') RETURNING *") + .describe("INSERT INTO tweet (id, text) VALUES (2, 'Hello') RETURNING *".into_sql_str()) .await .unwrap(); } async fn do_describe_insert_fks(db: &std::cell::RefCell) { db.borrow_mut() - .describe("insert into statements (text) values ('a') returning id") + .describe("insert into statements (text) values ('a') returning id".into_sql_str()) .await .unwrap(); } From c136ae9ea4e41a7bb96aa3b0c94c078c4507931b Mon Sep 17 00:00:00 2001 From: Keith Lohnes Date: Sun, 10 Aug 2025 13:33:47 -0400 Subject: [PATCH 18/19] Formatting --- benches/sqlite/describe.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/benches/sqlite/describe.rs b/benches/sqlite/describe.rs index 585bf9a4f9..e0e96ab1f0 100644 --- a/benches/sqlite/describe.rs +++ b/benches/sqlite/describe.rs @@ -3,13 +3,16 @@ use criterion::Criterion; use criterion::{criterion_group, criterion_main}; use sqlx::sqlite::{Sqlite, SqliteConnection}; -use sqlx::SqlSafeStr; use sqlx::Executor; +use sqlx::SqlSafeStr; use sqlx_test::new; // Here we have an async function to benchmark async fn do_describe_trivial(db: &std::cell::RefCell) { - db.borrow_mut().describe("select 1".into_sql_str()).await.unwrap(); + db.borrow_mut() + .describe("select 1".into_sql_str()) + .await + .unwrap(); } async fn do_describe_recursive(db: &std::cell::RefCell) { @@ -28,7 +31,8 @@ async fn do_describe_recursive(db: &std::cell::RefCell) { begin_date FROM schedule GROUP BY begin_date - "#.into_sql_str(), + "# + .into_sql_str(), ) .await .unwrap(); From d068e3b5ef764cc87653b8349d2718dea565de4a Mon Sep 17 00:00:00 2001 From: Keith Lohnes Date: Sun, 10 Aug 2025 13:38:03 -0400 Subject: [PATCH 19/19] Allow too many params for now --- sqlx-cli/src/migrate.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sqlx-cli/src/migrate.rs b/sqlx-cli/src/migrate.rs index b846ada18b..9018a57a92 100644 --- a/sqlx-cli/src/migrate.rs +++ b/sqlx-cli/src/migrate.rs @@ -213,6 +213,7 @@ fn validate_applied_migrations( Ok(()) } +#[allow(clippy::too_many_arguments)] pub async fn run( config: &Config, migration_source: &MigrationSourceOpt, @@ -337,6 +338,7 @@ pub async fn run( Ok(()) } +#[allow(clippy::too_many_arguments)] pub async fn revert( config: &Config, migration_source: &MigrationSourceOpt,