From 2305b22fbb1c34c02cc621dfe2d45c05a66162e0 Mon Sep 17 00:00:00 2001 From: "ashish.alex10@gmail.com" Date: Mon, 10 Nov 2025 15:31:32 +0000 Subject: [PATCH 1/2] feat: bring actions to be executed from compilation config --- src/dataformApi.ts | 9 +++ src/dataformApiUtils.ts | 170 +++++++++++++++++++++++++++++++++++++++- src/runCurrentFile.ts | 76 ++++++------------ src/types.ts | 1 + src/utils.ts | 9 ++- 5 files changed, 206 insertions(+), 59 deletions(-) diff --git a/src/dataformApi.ts b/src/dataformApi.ts index 185d2f7d..fa6e9603 100644 --- a/src/dataformApi.ts +++ b/src/dataformApi.ts @@ -90,6 +90,15 @@ export class DataformApi { await this.client.installNpmPackages(request); } + async queryCompilationResultActions(compilationResultName:string) { + const request = { + name: compilationResultName + }; + const [actions] = await this.client.queryCompilationResultActions(request); + return actions; + } + + /** * Pull commits from the remote git branch of the workspace. Git username and email are determined by git cli to set The author of any merge commit which may be created as a result of merging fetched Git commits into this workspace.. * diff --git a/src/dataformApiUtils.ts b/src/dataformApiUtils.ts index df9b52b1..6aaa8d94 100644 --- a/src/dataformApiUtils.ts +++ b/src/dataformApiUtils.ts @@ -3,7 +3,7 @@ import path from 'path'; import { getLocalGitState, getGitStatusCommitedFiles, gitRemoteBranchExsists} from "./getGitMeta"; import { getWorkspaceFolder, runCompilation, getGcpProjectLocationDataform} from './utils'; import { DataformApi } from './dataformApi'; -import { CreateCompilationResultResponse, InvocationConfig , GitFileChange, ICodeCompilationConfig} from "./types"; +import { CreateCompilationResultResponse, InvocationConfig , GitFileChange, ICodeCompilationConfig, CompilationType, ITarget} from "./types"; export function sendWorkflowInvocationNotification(url:string){ vscode.window.showInformationMessage( @@ -349,4 +349,172 @@ export async function syncAndrunDataformRemotely(progress: vscode.Progress<{ mes //7 progress.report({ message: 'Syncing remote workspace to local code...', increment: 14.28 }); await compileAndCreateWorkflowInvocation(dataformClient, invocationConfig, codeCompilationConfig); +}; + +export async function _syncAndrunDataformRemotely(progress: vscode.Progress<{ message?: string; increment?: number }>, token: vscode.CancellationToken, compilationType:CompilationType, relativeFilePath:string, includDependencies:boolean, includeDependents:boolean, fullRefresh:boolean, codeCompilationConfig?:ICodeCompilationConfig){ + // 1 + progress.report({ message: 'Checking for cached compilation of Dataform project...', increment: 0 }); + if (!CACHED_COMPILED_DATAFORM_JSON) { + if (token.isCancellationRequested) { + vscode.window.showInformationMessage('Operation cancelled during compilation check.'); + return; + } + + let workspaceFolder = await getWorkspaceFolder(); + if (!workspaceFolder) { + vscode.window.showErrorMessage('No workspace folder selected.'); + return; + } + + // 1 + progress.report({ message: 'Cache miss, compiling Dataform project...', increment: 14.28 }); + let { dataformCompiledJson } = await runCompilation(workspaceFolder); // ~1100ms + if (token.isCancellationRequested) { + vscode.window.showInformationMessage('Operation cancelled during compilation.'); + return; + } + + if (dataformCompiledJson) { + CACHED_COMPILED_DATAFORM_JSON = dataformCompiledJson; + } else { + vscode.window.showErrorMessage(`Unable to compile Dataform project. Run "dataform compile" in the terminal to check`); + return; + } + } + + if (token.isCancellationRequested) { + vscode.window.showInformationMessage('Operation cancelled during GCP validation.'); + return; + } + + const gcpProjectIdOveride = vscode.workspace.getConfiguration('vscode-dataform-tools').get('gcpProjectId'); + const gcpProjectId = (gcpProjectIdOveride || CACHED_COMPILED_DATAFORM_JSON.projectConfig.defaultDatabase) as string; + if (!gcpProjectId) { + vscode.window.showErrorMessage(`Unable to determine GCP project ID in Dataform config`); + return; + } + + let gcpProjectLocation = await getGcpProjectLocationDataform(gcpProjectId, CACHED_COMPILED_DATAFORM_JSON); + if (token.isCancellationRequested) { + vscode.window.showInformationMessage('Operation cancelled during GCP location fetch.'); + return; + } + + // 2 + progress.report({ message: 'Initializing Dataform client...', increment: 14.28 }); + const serviceAccountJsonPath = vscode.workspace.getConfiguration('vscode-dataform-tools').get('serviceAccountJsonPath'); + let clientOptions = { projectId: gcpProjectId }; + if(serviceAccountJsonPath){ + vscode.window.showInformationMessage(`Using service account at: ${serviceAccountJsonPath}`); + // @ts-ignore + clientOptions = {... clientOptions , keyFilename: serviceAccountJsonPath}; + } + + let options = { + clientOptions + }; + + const dataformClient = new DataformApi(gcpProjectId, gcpProjectLocation, options); + if (token.isCancellationRequested) { + vscode.window.showInformationMessage('Operation cancelled during client initialization.'); + return; + } + + if(compilationType === "workspace"){ + // 3 + progress.report({ message: `Creating Dataform workspace ${dataformClient.workspaceId} if it does not exsist...`, increment: 14.28 }); + try { + await dataformClient.createWorkspace(); + } catch (error: any) { + const DATAFORM_WORKSPACE_EXSIST_IN_GCP_ERROR_CODE = 6; + const DATAFORM_WORKSPACE_PARENT_NOT_FOUND_ERROR_CODE = 5; + + if (token.isCancellationRequested) { + vscode.window.showInformationMessage('Operation cancelled during workspace creation.'); + return; + } + + if (error.code === DATAFORM_WORKSPACE_EXSIST_IN_GCP_ERROR_CODE) { + // vscode.window.showWarningMessage(error.message); + } else if (error.code === DATAFORM_WORKSPACE_PARENT_NOT_FOUND_ERROR_CODE) { + error.message += `. Check if the Dataform repository ${dataformClient.gitRepoName} exists in GCP`; + vscode.window.showErrorMessage(error.message); + throw error; + } else { + vscode.window.showErrorMessage(error.message); + throw error; + } + } + + // 4 + progress.report({ message: `Verifying if git remote origin/${dataformClient.workspaceId} exsists...`, increment: 14.28 }); + let remoteGitRepoExsists = await gitRemoteBranchExsists(dataformClient.gitBranch); + if (token.isCancellationRequested) { + vscode.window.showInformationMessage('Operation cancelled during workflow execution.'); + return; + } + + if(remoteGitRepoExsists){ + // 5 + progress.report({ message: `Pulling Git commits into workspace ${dataformClient.workspaceId}...`, increment: 14.28 }); + try { + await dataformClient.pullGitCommits(); + } catch (error: any) { + //TODO: should we show user warning, and do a git resotore and pull changes ? + const CANNOT_PULL_UNCOMMITED_CHANGES_ERROR_CODE = 9; + //NOTE: this should not happen anymore as we are checking for git remote first + // const NO_REMOTE_ERROR_MSG = `9 FAILED_PRECONDITION: Could not pull branch '${dataformClient.workspaceId}' as it was not found remotely.`; + + if (token.isCancellationRequested) { + vscode.window.showInformationMessage('Operation cancelled during Git pull.'); + return; + } + if (error.code === CANNOT_PULL_UNCOMMITED_CHANGES_ERROR_CODE) { + vscode.window.showWarningMessage(error.message); + } else { + throw error; + } + } + } + + // 6 + progress.report({ message: 'Syncing remote workspace to local code...', increment: 14.28 }); + await syncRemoteWorkspaceToLocalBranch(dataformClient, remoteGitRepoExsists); + + //7 + progress.report({ message: 'Syncing remote workspace to local code...', increment: 14.28 }); + } + + + const compilationResult = await dataformClient.createCompilationResult(compilationType, codeCompilationConfig); + const fullCompilationResultName = compilationResult.name; + let actionsList: ITarget[] = []; + if(fullCompilationResultName){ + const actions = await dataformClient.queryCompilationResultActions(fullCompilationResultName); + actions.forEach((action) => { + if(action.filePath === relativeFilePath){ + if(action.target){ + actionsList.push(action.target); + } + } + }); + + if(actionsList.length < 1){ + //TODO: make error message better + vscode.window.showErrorMessage(`No actions found for ${relativeFilePath}`); + return; + } + + const invocationConfig = { + includedTargets: actionsList, + transitiveDependenciesIncluded: includDependencies, + transitiveDependentsIncluded: includeDependents, + fullyRefreshIncrementalTablesEnabled: fullRefresh, + }; + const createdWorkflowInvocation = await dataformClient.createDataformWorkflowInvocation(invocationConfig, fullCompilationResultName); + if(createdWorkflowInvocation?.url){ + sendWorkflowInvocationNotification(createdWorkflowInvocation.url); + } + return; + } }; \ No newline at end of file diff --git a/src/runCurrentFile.ts b/src/runCurrentFile.ts index cfc44afa..3dc04f5a 100644 --- a/src/runCurrentFile.ts +++ b/src/runCurrentFile.ts @@ -1,7 +1,6 @@ import * as vscode from 'vscode'; -import { getDataformActionCmdFromActionList, getDataformCompilationTimeoutFromConfig, getFileNameFromDocument, getQueryMetaForCurrentFile, getVSCodeDocument, getWorkspaceFolder, runCommandInTerminal, runCompilation, getLocationOfGcpProject, showLoadingProgress } from "./utils"; -import { DataformApi } from "./dataformApi"; -import { sendWorkflowInvocationNotification, syncAndrunDataformRemotely } from "./dataformApiUtils"; +import { getDataformActionCmdFromActionList, getDataformCompilationTimeoutFromConfig, getFileNameFromDocument, getQueryMetaForCurrentFile, getVSCodeDocument, getWorkspaceFolder, runCommandInTerminal, runCompilation, showLoadingProgress } from "./utils"; +import { _syncAndrunDataformRemotely } from "./dataformApiUtils"; import { ExecutionMode } from './types'; export async function runCurrentFile(includDependencies: boolean, includeDependents: boolean, fullRefresh: boolean, executionMode:ExecutionMode): Promise<{ workflowInvocationUrlGCP: string|undefined; errorWorkflowInvocation: string|undefined; } | undefined> { @@ -11,7 +10,12 @@ export async function runCurrentFile(includDependencies: boolean, includeDepende return; } - var result = getFileNameFromDocument(document, false); + let normalizeForWindows = true; + if (executionMode === "api" || executionMode === "api_workspace") { + normalizeForWindows = false; + } + + var result = getFileNameFromDocument(document, false, normalizeForWindows); if (result.success === false) { vscode.window.showErrorMessage(`Extension was unable to get filename of the current file`); return; @@ -55,57 +59,21 @@ export async function runCurrentFile(includDependencies: boolean, includeDepende dataformActionCmd = getDataformActionCmdFromActionList(actionsList, workspaceFolder, dataformCompilationTimeoutVal, includDependencies, includeDependents, fullRefresh); runCommandInTerminal(dataformActionCmd); return; - } else if (executionMode === "api" || executionMode === "api_workspace"){ - const projectId = CACHED_COMPILED_DATAFORM_JSON?.projectConfig.defaultDatabase; - if(!projectId){ - vscode.window.showErrorMessage("Unable to determine GCP project id to use for Dataform API run"); - return; - } - - let gcpProjectLocation = undefined; - if(CACHED_COMPILED_DATAFORM_JSON?.projectConfig.defaultLocation){ - gcpProjectLocation = CACHED_COMPILED_DATAFORM_JSON.projectConfig.defaultLocation; - }else{ - gcpProjectLocation = await getLocationOfGcpProject(projectId); - } - - if(!gcpProjectLocation){ - vscode.window.showErrorMessage("Unable to determine GCP project location to use for Dataform API run"); - return; - } - - let actionsList: {database:string, schema: string, name:string}[] = []; - currFileMetadata.tables.forEach((table: { target: { database: string; schema: string; name: string; }; }) => { - const action = {database: table.target.database, schema: table.target.schema, name: table.target.name}; - actionsList.push(action); - }); - - const invocationConfig = { - includedTargets: actionsList, - transitiveDependenciesIncluded: includDependencies, - transitiveDependentsIncluded: includeDependents, - fullyRefreshIncrementalTablesEnabled: fullRefresh, - }; - + } else if (executionMode === "api" || executionMode === "api_workspace") { + const compilationType = executionMode === "api" ? "gitBranch" : "workspace"; try{ - if(executionMode === "api_workspace"){ - await showLoadingProgress( - "", - syncAndrunDataformRemotely, - "Dataform remote workspace execution cancelled", - invocationConfig, - compilerOptionsMap, - ); - return; - } - const dataformClient = new DataformApi(projectId, gcpProjectLocation); - vscode.window.showInformationMessage(`Creating workflow invocation with ${dataformClient.gitBranch} remote git branch ...`); - const createdWorkflowInvocation = await dataformClient.runDataformRemotely(invocationConfig, "gitBranch", compilerOptionsMap); - const url = createdWorkflowInvocation?.url; - if(url){ - sendWorkflowInvocationNotification(url); - return {workflowInvocationUrlGCP: url, errorWorkflowInvocation: undefined}; - } + await showLoadingProgress( + "", + _syncAndrunDataformRemotely, + "Dataform remote workspace execution cancelled", + compilationType, + relativeFilePath, + includDependencies, + includeDependents, + fullRefresh, + compilerOptionsMap, + ); + return; } catch(error:any){ vscode.window.showErrorMessage(error.message); return {workflowInvocationUrlGCP: undefined, errorWorkflowInvocation: error.message}; diff --git a/src/types.ts b/src/types.ts index 44ca0cec..fbdcf5b9 100644 --- a/src/types.ts +++ b/src/types.ts @@ -362,6 +362,7 @@ export type CreateCompilationResultResponse = Promise< export type InvocationConfig = protos.google.cloud.dataform.v1beta1.IInvocationConfig; export type ICompilationResult = protos.google.cloud.dataform.v1beta1.ICompilationResult; export type ICodeCompilationConfig = protos.google.cloud.dataform.v1beta1.ICodeCompilationConfig; +export type ITarget = protos.google.cloud.dataform.v1beta1.ITarget; export type CompilationType = "gitBranch" | "workspace"; export type GitStatusCode = "M" | "A" | "??" | "D"; diff --git a/src/utils.ts b/src/utils.ts index f65bd040..fcb64b0c 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -919,10 +919,10 @@ export function debugExecutablePaths(): void { }); } -export function getRelativePath(filePath: string) { +export function getRelativePath(filePath: string, normalizeForWindow: boolean = true) { const fileUri = vscode.Uri.file(filePath); let relativePath = vscode.workspace.asRelativePath(fileUri); - if (isRunningOnWindows) { + if (isRunningOnWindows && normalizeForWindow) { relativePath = path.win32.normalize(relativePath); } const firstDefinitionIndex = relativePath.indexOf("definitions"); @@ -968,13 +968,14 @@ export async function selectWorkspaceFolder() { export function getFileNameFromDocument( document: vscode.TextDocument, - showErrorMessage: boolean + showErrorMessage: boolean, + normalizeForWindow: boolean = true ): FileNameMetadataResult { const filePath = document.uri.fsPath; const extWithDot = path.extname(filePath); const extension = extWithDot.startsWith('.') ? extWithDot.slice(1) : extWithDot; const rawFileName = path.basename(filePath, extWithDot); - const relativeFilePath = getRelativePath(filePath); + const relativeFilePath = getRelativePath(filePath, normalizeForWindow); const validFileType = supportedExtensions.includes(extension); if (!validFileType) { From a94ab921838eaed71df8321542f941a3b5ac6dbd Mon Sep 17 00:00:00 2001 From: "ashish.alex10@gmail.com" Date: Mon, 10 Nov 2025 22:06:20 +0000 Subject: [PATCH 2/2] feat: remove parts where i use cli to get compilation graph --- src/dataformApiUtils.ts | 55 +++++------------------------------- src/runCurrentFile.ts | 38 ++++++++++++------------- src/runFilesTagsWtOptions.ts | 4 +-- 3 files changed, 28 insertions(+), 69 deletions(-) diff --git a/src/dataformApiUtils.ts b/src/dataformApiUtils.ts index 6aaa8d94..5ace9c73 100644 --- a/src/dataformApiUtils.ts +++ b/src/dataformApiUtils.ts @@ -352,55 +352,9 @@ export async function syncAndrunDataformRemotely(progress: vscode.Progress<{ mes }; export async function _syncAndrunDataformRemotely(progress: vscode.Progress<{ message?: string; increment?: number }>, token: vscode.CancellationToken, compilationType:CompilationType, relativeFilePath:string, includDependencies:boolean, includeDependents:boolean, fullRefresh:boolean, codeCompilationConfig?:ICodeCompilationConfig){ - // 1 - progress.report({ message: 'Checking for cached compilation of Dataform project...', increment: 0 }); - if (!CACHED_COMPILED_DATAFORM_JSON) { - if (token.isCancellationRequested) { - vscode.window.showInformationMessage('Operation cancelled during compilation check.'); - return; - } - - let workspaceFolder = await getWorkspaceFolder(); - if (!workspaceFolder) { - vscode.window.showErrorMessage('No workspace folder selected.'); - return; - } - - // 1 - progress.report({ message: 'Cache miss, compiling Dataform project...', increment: 14.28 }); - let { dataformCompiledJson } = await runCompilation(workspaceFolder); // ~1100ms - if (token.isCancellationRequested) { - vscode.window.showInformationMessage('Operation cancelled during compilation.'); - return; - } - - if (dataformCompiledJson) { - CACHED_COMPILED_DATAFORM_JSON = dataformCompiledJson; - } else { - vscode.window.showErrorMessage(`Unable to compile Dataform project. Run "dataform compile" in the terminal to check`); - return; - } - } + const gcpProjectId = "drawingfire-b72a8"; + const gcpProjectLocation = "europe-west2"; - if (token.isCancellationRequested) { - vscode.window.showInformationMessage('Operation cancelled during GCP validation.'); - return; - } - - const gcpProjectIdOveride = vscode.workspace.getConfiguration('vscode-dataform-tools').get('gcpProjectId'); - const gcpProjectId = (gcpProjectIdOveride || CACHED_COMPILED_DATAFORM_JSON.projectConfig.defaultDatabase) as string; - if (!gcpProjectId) { - vscode.window.showErrorMessage(`Unable to determine GCP project ID in Dataform config`); - return; - } - - let gcpProjectLocation = await getGcpProjectLocationDataform(gcpProjectId, CACHED_COMPILED_DATAFORM_JSON); - if (token.isCancellationRequested) { - vscode.window.showInformationMessage('Operation cancelled during GCP location fetch.'); - return; - } - - // 2 progress.report({ message: 'Initializing Dataform client...', increment: 14.28 }); const serviceAccountJsonPath = vscode.workspace.getConfiguration('vscode-dataform-tools').get('serviceAccountJsonPath'); let clientOptions = { projectId: gcpProjectId }; @@ -487,6 +441,11 @@ export async function _syncAndrunDataformRemotely(progress: vscode.Progress<{ me const compilationResult = await dataformClient.createCompilationResult(compilationType, codeCompilationConfig); + if(compilationResult?.compilationErrors && compilationResult.compilationErrors.length >0){ + const errorMessages = compilationResult.compilationErrors.map((err) => {return err.message;}).join("; "); + vscode.window.showErrorMessage(errorMessages); + return; + } const fullCompilationResultName = compilationResult.name; let actionsList: ITarget[] = []; if(fullCompilationResultName){ diff --git a/src/runCurrentFile.ts b/src/runCurrentFile.ts index 3dc04f5a..2ad32419 100644 --- a/src/runCurrentFile.ts +++ b/src/runCurrentFile.ts @@ -26,31 +26,31 @@ export async function runCurrentFile(includDependencies: boolean, includeDepende if (!workspaceFolder) { return; } + if (executionMode === "cli") { - let dataformCompilationTimeoutVal = getDataformCompilationTimeoutFromConfig(); + let dataformCompilationTimeoutVal = getDataformCompilationTimeoutFromConfig(); - let currFileMetadata; - if (!CACHED_COMPILED_DATAFORM_JSON) { + let currFileMetadata; + if (!CACHED_COMPILED_DATAFORM_JSON) { - let {dataformCompiledJson, errors} = await runCompilation(workspaceFolder); // Takes ~1100ms - if(errors && errors.length > 0){ - vscode.window.showErrorMessage("Error compiling Dataform. Run `dataform compile` to see more details"); - return; - } - if (dataformCompiledJson) { - CACHED_COMPILED_DATAFORM_JSON = dataformCompiledJson; + let {dataformCompiledJson, errors} = await runCompilation(workspaceFolder); // Takes ~1100ms + if(errors && errors.length > 0){ + vscode.window.showErrorMessage("Error compiling Dataform. Run `dataform compile` to see more details"); + return; + } + if (dataformCompiledJson) { + CACHED_COMPILED_DATAFORM_JSON = dataformCompiledJson; + } } - } - if (CACHED_COMPILED_DATAFORM_JSON) { - currFileMetadata = await getQueryMetaForCurrentFile(relativeFilePath, CACHED_COMPILED_DATAFORM_JSON); - } - if(!currFileMetadata){ - vscode.window.showErrorMessage(`Unable to get metadata for the current file`); - return; - } + if (CACHED_COMPILED_DATAFORM_JSON) { + currFileMetadata = await getQueryMetaForCurrentFile(relativeFilePath, CACHED_COMPILED_DATAFORM_JSON); + } + if(!currFileMetadata){ + vscode.window.showErrorMessage(`Unable to get metadata for the current file`); + return; + } - if (executionMode === "cli") { let actionsList: string[] = currFileMetadata.tables.map(table => `${table.target.database}.${table.target.schema}.${table.target.name}`); let dataformActionCmd = ""; diff --git a/src/runFilesTagsWtOptions.ts b/src/runFilesTagsWtOptions.ts index 90059e1a..08d76c87 100644 --- a/src/runFilesTagsWtOptions.ts +++ b/src/runFilesTagsWtOptions.ts @@ -80,7 +80,7 @@ export async function runFilesTagsWtOptions(executionMode: ExecutionMode) { if (executionMode === "cli"){ if (firstStageSelection === "run current file") { - runCurrentFile(includeDependencies, includeDependents, fullRefresh, "cli"); + runCurrentFile(includeDependencies, includeDependents, fullRefresh, executionMode); } else if (firstStageSelection === "run a tag") { if(!tagSelection){return;}; let defaultDataformCompileTime = getDataformCompilationTimeoutFromConfig(); @@ -88,7 +88,7 @@ export async function runFilesTagsWtOptions(executionMode: ExecutionMode) { runCommandInTerminal(runTagsWtDepsCommand); } else if (firstStageSelection === "run multiple files"){ if(!multipleFileSelection){return;}; - runMultipleFilesFromSelection(workspaceFolder, multipleFileSelection, includeDependencies, includeDependents, fullRefresh, "cli"); + runMultipleFilesFromSelection(workspaceFolder, multipleFileSelection, includeDependencies, includeDependents, fullRefresh, executionMode); } else if (firstStageSelection === "run multiple tags"){ if(!multipleTagsSelection){return;}; runMultipleTagsFromSelection(workspaceFolder, multipleTagsSelection, includeDependencies, includeDependents, fullRefresh);