From 4641f46809594e55aeac2018f5e7af08f5a78df0 Mon Sep 17 00:00:00 2001 From: Guillaume JANVIER Date: Fri, 11 Jul 2025 12:04:46 +0200 Subject: [PATCH 1/2] feat(firestore-bigquery-exports) add an option to skip BigQuery init steps --- firestore-bigquery-export/scripts/import/src/config.ts | 9 +++++++++ firestore-bigquery-export/scripts/import/src/index.ts | 5 ++++- firestore-bigquery-export/scripts/import/src/program.ts | 4 ++++ firestore-bigquery-export/scripts/import/src/types.ts | 1 + 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/firestore-bigquery-export/scripts/import/src/config.ts b/firestore-bigquery-export/scripts/import/src/config.ts index b9f839027..45eea725a 100644 --- a/firestore-bigquery-export/scripts/import/src/config.ts +++ b/firestore-bigquery-export/scripts/import/src/config.ts @@ -197,6 +197,12 @@ const questions = [ name: "failedBatchOutput", type: "input", }, + { + message: "Would you like to skip BigQuery init steps?", + name: "skipInit", + type: "confirm", + default: false, + }, ]; export async function parseConfig(): Promise { @@ -270,6 +276,7 @@ export async function parseConfig(): Promise { rawChangeLogName, cursorPositionFile, failedBatchOutput: program.failedBatchOutput, + skipInit: program.skipInit, }; } const { @@ -285,6 +292,7 @@ export async function parseConfig(): Promise { useNewSnapshotQuerySyntax, useEmulator, failedBatchOutput, + skipInit, } = await inquirer.prompt(questions); const rawChangeLogName = `${table}_raw_changelog`; @@ -311,6 +319,7 @@ export async function parseConfig(): Promise { rawChangeLogName, cursorPositionFile, failedBatchOutput, + skipInit, }; } diff --git a/firestore-bigquery-export/scripts/import/src/index.ts b/firestore-bigquery-export/scripts/import/src/index.ts index 0b9b3d439..016653e58 100644 --- a/firestore-bigquery-export/scripts/import/src/index.ts +++ b/firestore-bigquery-export/scripts/import/src/index.ts @@ -82,9 +82,12 @@ const run = async (): Promise => { useNewSnapshotQuerySyntax, bqProjectId: bigQueryProjectId, transformFunction: config.transformFunctionUrl, + skipInit: config.skipInit, }); - await initializeDataSink(dataSink, config); + if (!config.skipInit) { + await initializeDataSink(dataSink, config); + } logs.importingData(config); if (multiThreaded && queryCollectionGroup) { diff --git a/firestore-bigquery-export/scripts/import/src/program.ts b/firestore-bigquery-export/scripts/import/src/program.ts index 87982aa16..6cf3dc98f 100644 --- a/firestore-bigquery-export/scripts/import/src/program.ts +++ b/firestore-bigquery-export/scripts/import/src/program.ts @@ -65,5 +65,9 @@ export const getCLIOptions = () => { .option( "-f, --failed-batch-output ", "Path to the JSON file where failed batches will be recorded." + ) + .option( + "--skip-init [true|false]", + "Whether to skip BigQuery init steps." ); }; diff --git a/firestore-bigquery-export/scripts/import/src/types.ts b/firestore-bigquery-export/scripts/import/src/types.ts index e2dfdcd18..3511666e0 100644 --- a/firestore-bigquery-export/scripts/import/src/types.ts +++ b/firestore-bigquery-export/scripts/import/src/types.ts @@ -17,6 +17,7 @@ export interface CliConfig { cursorPositionFile: string; failedBatchOutput?: string; transformFunctionUrl?: string; + skipInit: boolean; } export interface CliConfigError { From 63f969f84566072c295379f6160c48ecac9b79a6 Mon Sep 17 00:00:00 2001 From: Guillaume JANVIER Date: Tue, 28 Oct 2025 09:35:08 +0100 Subject: [PATCH 2/2] fix: apply recommanded changes --- firestore-bigquery-export/scripts/import/src/config.ts | 2 +- firestore-bigquery-export/scripts/import/src/program.ts | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/firestore-bigquery-export/scripts/import/src/config.ts b/firestore-bigquery-export/scripts/import/src/config.ts index 45eea725a..bce16c70c 100644 --- a/firestore-bigquery-export/scripts/import/src/config.ts +++ b/firestore-bigquery-export/scripts/import/src/config.ts @@ -276,7 +276,7 @@ export async function parseConfig(): Promise { rawChangeLogName, cursorPositionFile, failedBatchOutput: program.failedBatchOutput, - skipInit: program.skipInit, + skipInit: program.skipInit === true || program.skipInit === 'true', }; } const { diff --git a/firestore-bigquery-export/scripts/import/src/program.ts b/firestore-bigquery-export/scripts/import/src/program.ts index 6cf3dc98f..f90986406 100644 --- a/firestore-bigquery-export/scripts/import/src/program.ts +++ b/firestore-bigquery-export/scripts/import/src/program.ts @@ -67,7 +67,8 @@ export const getCLIOptions = () => { "Path to the JSON file where failed batches will be recorded." ) .option( - "--skip-init [true|false]", - "Whether to skip BigQuery init steps." + "--skip-init", + "Whether to skip BigQuery init steps.", + false ); };