feat: bring actions to be executed from api compilation result

src/dataformApi.ts

@@ -90,6 +90,15 @@ export class DataformApi {
         await this.client.installNpmPackages(request);
     }
 
+    async queryCompilationResultActions(compilationResultName:string) {
+        const request = {
+            name: compilationResultName
+        };
+        const [actions] = await this.client.queryCompilationResultActions(request);
+        return actions;
+    }
+
+
     /**
      * Pull commits from the remote git branch of the workspace. Git username and email are determined by git cli to set The author of any merge commit which may be created as a result of merging fetched Git commits into this workspace..
      *

src/dataformApiUtils.ts

@@ -3,7 +3,7 @@ import path from 'path';
 import { getLocalGitState, getGitStatusCommitedFiles, gitRemoteBranchExsists} from "./getGitMeta";
 import { getWorkspaceFolder, runCompilation, getGcpProjectLocationDataform} from './utils';
 import { DataformApi } from './dataformApi';
-import { CreateCompilationResultResponse, InvocationConfig , GitFileChange, ICodeCompilationConfig} from "./types";
+import { CreateCompilationResultResponse, InvocationConfig , GitFileChange, ICodeCompilationConfig, CompilationType, ITarget} from "./types";
 
 export function sendWorkflowInvocationNotification(url:string){
     vscode.window.showInformationMessage(
@@ -349,4 +349,131 @@ export async function syncAndrunDataformRemotely(progress: vscode.Progress<{ mes
         //7
         progress.report({ message: 'Syncing remote workspace to local code...', increment: 14.28 });
         await compileAndCreateWorkflowInvocation(dataformClient, invocationConfig, codeCompilationConfig);
+};
+
+export async function _syncAndrunDataformRemotely(progress: vscode.Progress<{ message?: string; increment?: number }>, token: vscode.CancellationToken, compilationType:CompilationType, relativeFilePath:string, includDependencies:boolean, includeDependents:boolean, fullRefresh:boolean, codeCompilationConfig?:ICodeCompilationConfig){
+        const gcpProjectId = "drawingfire-b72a8";
+        const gcpProjectLocation = "europe-west2";
+
+        progress.report({ message: 'Initializing Dataform client...', increment: 14.28 });
+        const serviceAccountJsonPath  = vscode.workspace.getConfiguration('vscode-dataform-tools').get('serviceAccountJsonPath');
+        let clientOptions = { projectId: gcpProjectId };
+        if(serviceAccountJsonPath){
+            vscode.window.showInformationMessage(`Using service account at: ${serviceAccountJsonPath}`);
+            // @ts-ignore 
+            clientOptions = {... clientOptions , keyFilename: serviceAccountJsonPath};
+        }
+
+        let options = {
+            clientOptions
+        };
+
+        const dataformClient = new DataformApi(gcpProjectId, gcpProjectLocation, options);
+        if (token.isCancellationRequested) {
+            vscode.window.showInformationMessage('Operation cancelled during client initialization.');
+            return;
+        }
+
+        if(compilationType === "workspace"){
+            // 3
+            progress.report({ message: `Creating Dataform workspace ${dataformClient.workspaceId} if it does not exsist...`, increment: 14.28 });
+            try {
+                await dataformClient.createWorkspace();
+            } catch (error: any) {
+                const DATAFORM_WORKSPACE_EXSIST_IN_GCP_ERROR_CODE = 6;
+                const DATAFORM_WORKSPACE_PARENT_NOT_FOUND_ERROR_CODE = 5;
+
+                if (token.isCancellationRequested) {
+                    vscode.window.showInformationMessage('Operation cancelled during workspace creation.');
+                    return;
+                }
+
+                if (error.code === DATAFORM_WORKSPACE_EXSIST_IN_GCP_ERROR_CODE) {
+                    // vscode.window.showWarningMessage(error.message);
+                } else if (error.code === DATAFORM_WORKSPACE_PARENT_NOT_FOUND_ERROR_CODE) {
+                    error.message += `. Check if the Dataform repository ${dataformClient.gitRepoName} exists in GCP`;
+                    vscode.window.showErrorMessage(error.message);
+                    throw error;
+                } else {
+                    vscode.window.showErrorMessage(error.message);
+                    throw error;
+                }
+            }
+
+            // 4
+            progress.report({ message: `Verifying if git remote origin/${dataformClient.workspaceId} exsists...`, increment: 14.28 });
+            let remoteGitRepoExsists = await gitRemoteBranchExsists(dataformClient.gitBranch);
+            if (token.isCancellationRequested) {
+                vscode.window.showInformationMessage('Operation cancelled during workflow execution.');
+                return;
+            }
+
+            if(remoteGitRepoExsists){
+                // 5
+                progress.report({ message: `Pulling Git commits into workspace ${dataformClient.workspaceId}...`, increment: 14.28 });
+                try {
+                    await dataformClient.pullGitCommits();
+                } catch (error: any) {
+                    //TODO: should we show user warning, and do a git resotore and pull changes ? 
+                    const CANNOT_PULL_UNCOMMITED_CHANGES_ERROR_CODE = 9;
+                    //NOTE: this should not happen anymore as we are checking for git remote first
+                    // const NO_REMOTE_ERROR_MSG = `9 FAILED_PRECONDITION: Could not pull branch '${dataformClient.workspaceId}' as it was not found remotely.`;
+
+                    if (token.isCancellationRequested) {
+                        vscode.window.showInformationMessage('Operation cancelled during Git pull.');
+                        return;
+                    }
+                    if (error.code === CANNOT_PULL_UNCOMMITED_CHANGES_ERROR_CODE) {
+                        vscode.window.showWarningMessage(error.message);
+                    } else {
+                        throw error;
+                    }
+                }
+            }
+
+            // 6
+            progress.report({ message: 'Syncing remote workspace to local code...', increment: 14.28 });
+            await syncRemoteWorkspaceToLocalBranch(dataformClient, remoteGitRepoExsists);
+
+            //7
+            progress.report({ message: 'Syncing remote workspace to local code...', increment: 14.28 });
+        }
+
+
+        const compilationResult = await dataformClient.createCompilationResult(compilationType, codeCompilationConfig);
+        if(compilationResult?.compilationErrors && compilationResult.compilationErrors.length >0){
+            const errorMessages = compilationResult.compilationErrors.map((err) => {return err.message;}).join("; ");
+            vscode.window.showErrorMessage(errorMessages);
+            return;
+        }
+        const fullCompilationResultName = compilationResult.name;
+        let actionsList: ITarget[] = [];
+        if(fullCompilationResultName){
+            const actions = await dataformClient.queryCompilationResultActions(fullCompilationResultName);
+            actions.forEach((action) => {
+                if(action.filePath === relativeFilePath){
+                    if(action.target){
+                    actionsList.push(action.target);
+                    }
+                }
+            });
+
+            if(actionsList.length < 1){
+                //TODO: make error message better
+                vscode.window.showErrorMessage(`No actions found for ${relativeFilePath}`);
+                return;
+            }
+
+            const invocationConfig = {
+                includedTargets: actionsList,
+                transitiveDependenciesIncluded: includDependencies,
+                transitiveDependentsIncluded: includeDependents,
+                fullyRefreshIncrementalTablesEnabled: fullRefresh,
+            };
+            const createdWorkflowInvocation = await dataformClient.createDataformWorkflowInvocation(invocationConfig, fullCompilationResultName);
+            if(createdWorkflowInvocation?.url){
+                sendWorkflowInvocationNotification(createdWorkflowInvocation.url);
+            }
+            return;
+        }
 };

src/runCurrentFile.ts

@@ -1,7 +1,6 @@
 import * as vscode from 'vscode';
-import { getDataformActionCmdFromActionList, getDataformCompilationTimeoutFromConfig, getFileNameFromDocument, getQueryMetaForCurrentFile, getVSCodeDocument, getWorkspaceFolder, runCommandInTerminal, runCompilation, getLocationOfGcpProject, showLoadingProgress } from "./utils";
-import { DataformApi } from "./dataformApi";
-import { sendWorkflowInvocationNotification, syncAndrunDataformRemotely } from "./dataformApiUtils";
+import { getDataformActionCmdFromActionList, getDataformCompilationTimeoutFromConfig, getFileNameFromDocument, getQueryMetaForCurrentFile, getVSCodeDocument, getWorkspaceFolder, runCommandInTerminal, runCompilation, showLoadingProgress } from "./utils";
+import { _syncAndrunDataformRemotely } from "./dataformApiUtils";
 import { ExecutionMode } from './types';
 
 export async function runCurrentFile(includDependencies: boolean, includeDependents: boolean, fullRefresh: boolean, executionMode:ExecutionMode): Promise<{ workflowInvocationUrlGCP: string|undefined; errorWorkflowInvocation: string|undefined; } | undefined> {
@@ -11,7 +10,12 @@ export async function runCurrentFile(includDependencies: boolean, includeDepende
         return;
     }
 
-    var result = getFileNameFromDocument(document, false);
+    let normalizeForWindows = true;
+    if (executionMode === "api" || executionMode === "api_workspace") {
+        normalizeForWindows = false;
+    }
+
+    var result = getFileNameFromDocument(document, false, normalizeForWindows);
     if (result.success === false) {
         vscode.window.showErrorMessage(`Extension was unable to get filename of the current file`);
         return;
@@ -22,31 +26,31 @@ export async function runCurrentFile(includDependencies: boolean, includeDepende
     if (!workspaceFolder) {
         return;
     }
+    if (executionMode === "cli") {
 
-    let dataformCompilationTimeoutVal = getDataformCompilationTimeoutFromConfig();
+        let dataformCompilationTimeoutVal = getDataformCompilationTimeoutFromConfig();
 
-    let currFileMetadata;
-    if (!CACHED_COMPILED_DATAFORM_JSON) {
+        let currFileMetadata;
+        if (!CACHED_COMPILED_DATAFORM_JSON) {
 
-        let {dataformCompiledJson, errors} = await runCompilation(workspaceFolder); // Takes ~1100ms
-        if(errors && errors.length > 0){
-            vscode.window.showErrorMessage("Error compiling Dataform. Run `dataform compile` to see more details");
-            return;
-        }
-        if (dataformCompiledJson) {
-            CACHED_COMPILED_DATAFORM_JSON = dataformCompiledJson;
+            let {dataformCompiledJson, errors} = await runCompilation(workspaceFolder); // Takes ~1100ms
+            if(errors && errors.length > 0){
+                vscode.window.showErrorMessage("Error compiling Dataform. Run `dataform compile` to see more details");
+                return;
+            }
+            if (dataformCompiledJson) {
+                CACHED_COMPILED_DATAFORM_JSON = dataformCompiledJson;
+            }
         }
-    }
 
-    if (CACHED_COMPILED_DATAFORM_JSON) {
-        currFileMetadata = await getQueryMetaForCurrentFile(relativeFilePath, CACHED_COMPILED_DATAFORM_JSON);
-    }
-    if(!currFileMetadata){
-        vscode.window.showErrorMessage(`Unable to get metadata for the current file`);
-        return;
-    }
+        if (CACHED_COMPILED_DATAFORM_JSON) {
+            currFileMetadata = await getQueryMetaForCurrentFile(relativeFilePath, CACHED_COMPILED_DATAFORM_JSON);
+        }
+        if(!currFileMetadata){
+            vscode.window.showErrorMessage(`Unable to get metadata for the current file`);
+            return;
+        }
 
-    if (executionMode === "cli") {
         let actionsList: string[] = currFileMetadata.tables.map(table => `${table.target.database}.${table.target.schema}.${table.target.name}`);
 
         let dataformActionCmd = "";
@@ -55,57 +59,21 @@ export async function runCurrentFile(includDependencies: boolean, includeDepende
         dataformActionCmd = getDataformActionCmdFromActionList(actionsList, workspaceFolder, dataformCompilationTimeoutVal, includDependencies, includeDependents, fullRefresh);
         runCommandInTerminal(dataformActionCmd);
         return;
-    } else if (executionMode === "api" || executionMode === "api_workspace"){
-        const projectId = CACHED_COMPILED_DATAFORM_JSON?.projectConfig.defaultDatabase;
-        if(!projectId){
-            vscode.window.showErrorMessage("Unable to determine GCP project id to use for Dataform API run");
-            return;
-        }
-
-        let gcpProjectLocation = undefined;
-        if(CACHED_COMPILED_DATAFORM_JSON?.projectConfig.defaultLocation){
-            gcpProjectLocation = CACHED_COMPILED_DATAFORM_JSON.projectConfig.defaultLocation;
-        }else{
-            gcpProjectLocation = await getLocationOfGcpProject(projectId);
-        }
-
-        if(!gcpProjectLocation){
-            vscode.window.showErrorMessage("Unable to determine GCP project location to use for Dataform API run");
-            return;
-        }
-
-        let actionsList: {database:string, schema: string, name:string}[] = [];
-        currFileMetadata.tables.forEach((table: { target: { database: string; schema: string; name: string; }; }) => {
-            const action = {database: table.target.database, schema: table.target.schema, name: table.target.name};
-            actionsList.push(action);
-        });
-
-        const invocationConfig = {
-            includedTargets: actionsList,
-            transitiveDependenciesIncluded: includDependencies,
-            transitiveDependentsIncluded: includeDependents,
-            fullyRefreshIncrementalTablesEnabled: fullRefresh,
-        };
-
+    } else if (executionMode === "api" || executionMode === "api_workspace") {
+        const compilationType = executionMode  === "api" ? "gitBranch" : "workspace";
         try{
-            if(executionMode === "api_workspace"){
-                await showLoadingProgress(
-                    "",
-                    syncAndrunDataformRemotely,
-                    "Dataform remote workspace execution cancelled",
-                    invocationConfig,
-                    compilerOptionsMap,
-                );
-                return;
-            }
-            const dataformClient = new DataformApi(projectId, gcpProjectLocation);
-            vscode.window.showInformationMessage(`Creating workflow invocation with ${dataformClient.gitBranch} remote git branch ...`);
-            const createdWorkflowInvocation = await dataformClient.runDataformRemotely(invocationConfig, "gitBranch", compilerOptionsMap);
-            const url = createdWorkflowInvocation?.url;
-            if(url){
-                sendWorkflowInvocationNotification(url);
-                return {workflowInvocationUrlGCP: url, errorWorkflowInvocation: undefined};
-            }
+            await showLoadingProgress(
+                "",
+                _syncAndrunDataformRemotely,
+                "Dataform remote workspace execution cancelled",
+                compilationType,
+                relativeFilePath,
+                includDependencies,
+                includeDependents,
+                fullRefresh,
+                compilerOptionsMap,
+            );
+            return;
         } catch(error:any){
             vscode.window.showErrorMessage(error.message);
             return {workflowInvocationUrlGCP: undefined, errorWorkflowInvocation: error.message};

src/runFilesTagsWtOptions.ts

@@ -80,15 +80,15 @@ export async function runFilesTagsWtOptions(executionMode: ExecutionMode) {
 
     if (executionMode === "cli"){
         if (firstStageSelection === "run current file") {
-            runCurrentFile(includeDependencies, includeDependents, fullRefresh, "cli");
+            runCurrentFile(includeDependencies, includeDependents, fullRefresh, executionMode);
         } else if (firstStageSelection === "run a tag") {
             if(!tagSelection){return;};
             let defaultDataformCompileTime = getDataformCompilationTimeoutFromConfig();
             let runTagsWtDepsCommand = getRunTagsWtOptsCommand(workspaceFolder, [tagSelection], defaultDataformCompileTime, includeDependents, includeDependents, fullRefresh);
             runCommandInTerminal(runTagsWtDepsCommand);
         } else if (firstStageSelection === "run multiple files"){
             if(!multipleFileSelection){return;};
-            runMultipleFilesFromSelection(workspaceFolder, multipleFileSelection, includeDependencies, includeDependents, fullRefresh, "cli");
+            runMultipleFilesFromSelection(workspaceFolder, multipleFileSelection, includeDependencies, includeDependents, fullRefresh, executionMode);
         } else if (firstStageSelection === "run multiple tags"){
             if(!multipleTagsSelection){return;};
             runMultipleTagsFromSelection(workspaceFolder, multipleTagsSelection, includeDependencies, includeDependents, fullRefresh);

src/types.ts

@@ -362,6 +362,7 @@ export type CreateCompilationResultResponse = Promise<
 export type InvocationConfig = protos.google.cloud.dataform.v1beta1.IInvocationConfig;
 export type ICompilationResult  = protos.google.cloud.dataform.v1beta1.ICompilationResult;
 export type ICodeCompilationConfig = protos.google.cloud.dataform.v1beta1.ICodeCompilationConfig;
+export type ITarget = protos.google.cloud.dataform.v1beta1.ITarget;
 
 export type CompilationType  = "gitBranch" | "workspace";
 export type GitStatusCode = "M" | "A" | "??" | "D";

src/utils.ts

@@ -919,10 +919,10 @@ export function debugExecutablePaths(): void {
     });
 }
 
-export function getRelativePath(filePath: string) {
+export function getRelativePath(filePath: string, normalizeForWindow: boolean = true) {
     const fileUri = vscode.Uri.file(filePath);
     let relativePath = vscode.workspace.asRelativePath(fileUri);
-    if (isRunningOnWindows) {
+    if (isRunningOnWindows && normalizeForWindow) {
         relativePath = path.win32.normalize(relativePath);
     }
     const firstDefinitionIndex = relativePath.indexOf("definitions");
@@ -968,13 +968,14 @@ export async function selectWorkspaceFolder() {
 
 export function getFileNameFromDocument(
     document: vscode.TextDocument,
-    showErrorMessage: boolean
+    showErrorMessage: boolean,
+    normalizeForWindow: boolean = true
 ): FileNameMetadataResult<FileNameMetadata, string> {
     const filePath = document.uri.fsPath;
     const extWithDot = path.extname(filePath);
     const extension = extWithDot.startsWith('.') ? extWithDot.slice(1) : extWithDot;
     const rawFileName = path.basename(filePath, extWithDot);
-    const relativeFilePath = getRelativePath(filePath);
+    const relativeFilePath = getRelativePath(filePath, normalizeForWindow);
     const validFileType = supportedExtensions.includes(extension);
 
     if (!validFileType) {