From eb4302ba6984965a5e7974252c7bb76210a8e696 Mon Sep 17 00:00:00 2001 From: Bing64 <625255264@qq.com> Date: Sat, 2 Dec 2023 16:07:45 +0800 Subject: [PATCH 01/18] Add multi language support for CreateWorkflow and UpdateWorkflow functions --- internal/context/workspace/domain/workflow/consts.go | 7 +++++++ internal/context/workspace/domain/workflow/factory.go | 4 ++++ .../workspace/interface/hertz/handlers/workflow_vo.go | 4 ++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/internal/context/workspace/domain/workflow/consts.go b/internal/context/workspace/domain/workflow/consts.go index e507958..4b5896d 100644 --- a/internal/context/workspace/domain/workflow/consts.go +++ b/internal/context/workspace/domain/workflow/consts.go @@ -17,6 +17,13 @@ const ( WorkflowGitToken = "gitToken" ) +const ( + WorkflowLanguageWDL = "WDL" + WorkflowLanguageCWL = "CWL" + WorkflowLanguageSnakemake = "SMK" + WorkflowLanguageNextflow = "NFL" +) + const ( Language = "WDL" VersionRegexpStr = "^version\\s+([\\w-._]+)" diff --git a/internal/context/workspace/domain/workflow/factory.go b/internal/context/workspace/domain/workflow/factory.go index b4d44ca..dd049a9 100644 --- a/internal/context/workspace/domain/workflow/factory.go +++ b/internal/context/workspace/domain/workflow/factory.go @@ -83,6 +83,10 @@ func (p VersionOption) validate() error { return apperrors.NewInvalidError("tag") } } + // 增加language validation + if p.Language != WorkflowLanguageCWL && p.Language != WorkflowLanguageWDL && p.Language != WorkflowLanguageNextflow && p.Language != WorkflowLanguageSnakemake { + return apperrors.NewInvalidError("Language") + } return nil } diff --git a/internal/context/workspace/interface/hertz/handlers/workflow_vo.go b/internal/context/workspace/interface/hertz/handlers/workflow_vo.go index 2f8b77c..07ff76b 100644 --- a/internal/context/workspace/interface/hertz/handlers/workflow_vo.go +++ b/internal/context/workspace/interface/hertz/handlers/workflow_vo.go @@ -15,7 +15,7 @@ type createWorkflowRequest struct { WorkspaceID string `path:"workspace-id"` Name string `json:"name" validate:"required,resName"` Description *string `json:"description" validate:"workspaceDesc"` - Language string `json:"language" validate:"required,oneof=WDL"` + Language string `json:"language" validate:"required,oneof=WDL CWL NFL SMK"` Source string `json:"source" validate:"required,oneof=git"` URL string `json:"url" validate:"required"` Tag string `json:"tag" validate:"required"` @@ -101,7 +101,7 @@ type updateWorkflowRequest struct { ID string `path:"id"` Name *string `json:"name,omitempty"` Description *string `json:"description" validate:"workspaceDesc"` - Language *string `json:"language,omitempty" validate:"required,oneof=WDL"` + Language *string `json:"language,omitempty" validate:"required,oneof=WDL CWL NFL SMK"` Source *string `json:"source,omitempty" validate:"required,oneof=git"` URL *string `json:"url,omitempty"` Tag *string `json:"tag,omitempty"` From 5f184a396e35708fa2522412b3eb39fbf14a4795 Mon Sep 17 00:00:00 2001 From: Bing64 <625255264@qq.com> Date: Sat, 2 Dec 2023 23:46:36 +0800 Subject: [PATCH 02/18] Add package workflowparser, change workflow validation method --- .../application/command/workflow/commands.go | 9 +- .../domain/workflow/event_handlers.go | 214 +++--------------- .../workspace/domain/workflow/object.go | 21 ++ .../workspace/domain/workflow/service.go | 8 +- .../infrastructure/workflowparser/object.go | 65 ++++++ .../workflowparser/wdlparser.go | 200 ++++++++++++++++ .../workflowparser/workflowparser.go | 77 +++++++ 7 files changed, 411 insertions(+), 183 deletions(-) create mode 100644 internal/context/workspace/infrastructure/workflowparser/object.go create mode 100644 internal/context/workspace/infrastructure/workflowparser/wdlparser.go create mode 100644 internal/context/workspace/infrastructure/workflowparser/workflowparser.go diff --git a/internal/context/workspace/application/command/workflow/commands.go b/internal/context/workspace/application/command/workflow/commands.go index 29e62e7..7b5cfcd 100644 --- a/internal/context/workspace/application/command/workflow/commands.go +++ b/internal/context/workspace/application/command/workflow/commands.go @@ -5,6 +5,7 @@ import ( "github.com/Bio-OS/bioos/internal/context/workspace/application/query/workspace" "github.com/Bio-OS/bioos/internal/context/workspace/domain/workflow" "github.com/Bio-OS/bioos/internal/context/workspace/infrastructure/eventbus" + "github.com/Bio-OS/bioos/internal/context/workspace/infrastructure/workflowparser" ) type Commands struct { @@ -14,7 +15,13 @@ type Commands struct { } func NewCommands(repo workflow.Repository, workflowReadModel workflowquery.ReadModel, factory *workflow.Factory, workspaceReadModel workspace.WorkspaceReadModel, bus eventbus.EventBus, womtoolPath string) *Commands { - service := workflow.NewService(repo, workflowReadModel, bus, factory, womtoolPath) + service := workflow.NewService(repo, workflowReadModel, bus, factory) + configs := map[string]workflowparser.ParserConfig{ + workflowparser.WDL: workflowparser.WDLConfig{ + WomtoolPath: womtoolPath, + }, + } + workflowparser.InitWorkflowParserFactory(configs) return &Commands{ Create: NewCreateHandler(service, workflowReadModel, workspaceReadModel), Delete: NewDeleteHandler(service, workspaceReadModel), diff --git a/internal/context/workspace/domain/workflow/event_handlers.go b/internal/context/workspace/domain/workflow/event_handlers.go index 9023c58..25e3d34 100644 --- a/internal/context/workspace/domain/workflow/event_handlers.go +++ b/internal/context/workspace/domain/workflow/event_handlers.go @@ -1,17 +1,10 @@ package workflow import ( - "bufio" "context" - "encoding/base64" - "encoding/json" "fmt" "os" "path" - "path/filepath" - "regexp" - "sort" - "strings" "time" "k8s.io/apimachinery/pkg/util/sets" @@ -19,28 +12,22 @@ import ( "github.com/Bio-OS/bioos/internal/context/workspace/application/query/workflow" "github.com/Bio-OS/bioos/internal/context/workspace/domain/workspace" "github.com/Bio-OS/bioos/internal/context/workspace/infrastructure/eventbus" + "github.com/Bio-OS/bioos/internal/context/workspace/infrastructure/workflowparser" "github.com/Bio-OS/bioos/internal/context/workspace/interface/grpc/proto" apperrors "github.com/Bio-OS/bioos/pkg/errors" applog "github.com/Bio-OS/bioos/pkg/log" "github.com/Bio-OS/bioos/pkg/schema" - "github.com/Bio-OS/bioos/pkg/utils/exec" "github.com/Bio-OS/bioos/pkg/utils/git" "github.com/Bio-OS/bioos/pkg/validator" ) -const ( - CommandExecuteTimeout = time.Minute * 3 -) - type WorkflowVersionAddedHandler struct { - repo Repository - womtoolPath string + repo Repository } -func NewWorkflowVersionAddedHandler(repo Repository, womtoolPath string) *WorkflowVersionAddedHandler { +func NewWorkflowVersionAddedHandler(repo Repository) *WorkflowVersionAddedHandler { return &WorkflowVersionAddedHandler{ - repo: repo, - womtoolPath: womtoolPath, + repo: repo, } } @@ -84,6 +71,12 @@ func (h *WorkflowVersionAddedHandler) Handle(ctx context.Context, event *Workflo func (h *WorkflowVersionAddedHandler) handle(ctx context.Context, workflowID string, version *WorkflowVersion, event *WorkflowVersionAddedEvent) error { var dir string var err error + workflowParser, err := workflowparser.GetFactory().GetParser(version.Language) + if err != nil { + applog.Errorf("wrong workflow parser: %s", err) + return err + } + if event.FilesBaseDir != "" { dir = event.FilesBaseDir } else { @@ -113,194 +106,59 @@ func (h *WorkflowVersionAddedHandler) handle(ctx context.Context, workflowID str return apperrors.NewInternalError(err) } // parse workfile version - languageVersion, err := h.parseWorkflowVersion(ctx, mainWorkflowPath) + languageVersion, err := workflowParser.ParseWorkflowVersion(ctx, mainWorkflowPath) if err != nil { return apperrors.NewInternalError(err) } - version.Language = Language version.LanguageVersion = languageVersion // step3: validate and save workflow files - if err := h.validateWorkflowFiles(ctx, version, dir, version.MainWorkflowPath); err != nil { - return err - } - - // step4: get workflow inputs - inputs, err := h.getWorkflowInputs(ctx, mainWorkflowPath) - if err != nil { - return err - } - version.Inputs = inputs - - // step5: get workflow outputs - outputs, err := h.getWorkflowOutputs(ctx, mainWorkflowPath) + fileParamsStr, err := workflowParser.ValidateWorkflowFiles(ctx, dir, version.MainWorkflowPath) if err != nil { return err } - version.Outputs = outputs - - // step6: get workflow graph - graph, err := h.getWorkflowGraph(ctx, mainWorkflowPath) + fileParams, err := fileParamPOToDO(fileParamsStr) if err != nil { return err } - version.Graph = graph - return nil -} - -func (h *WorkflowVersionAddedHandler) parseWorkflowVersion(_ context.Context, mainWorkflowPath string) (string, error) { - versionRegexp := regexp.MustCompile(VersionRegexpStr) - file, err := os.Open(mainWorkflowPath) - if err != nil { - applog.Errorw("fail to open main workflow file", "err", err) - return "", err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - matched := versionRegexp.FindStringSubmatch(line) - if matched != nil && len(matched) >= 2 { - applog.Infow("version regexp matched", "matched", matched) - return matched[1], nil - } - } - - if err := scanner.Err(); err != nil { - return "", err - } - return "draft-2", nil -} - -func (h *WorkflowVersionAddedHandler) validateWorkflowFiles(ctx context.Context, version *WorkflowVersion, baseDir, mainWorkflowPath string) error { - applog.Infow("start to validate files", "mainWorkflowPath", mainWorkflowPath) - validateResult, err := exec.Exec(ctx, CommandExecuteTimeout, "java", "-jar", h.womtoolPath, "validate", path.Join(baseDir, mainWorkflowPath), "-l") - if err != nil { - applog.Errorw("fail to validate workflow", "err", err, "result", string(validateResult)) - return apperrors.NewInternalError(fmt.Errorf("validate workflow failed")) - } - validateResultLines := strings.Split(string(validateResult), "\n") - applog.Infow("validate result", "result", validateResultLines) - // parse and save workflow files - if len(validateResultLines) < 2 || strings.ToLower(validateResultLines[0]) != "success!" { - return proto.ErrorWorkflowValidateError("fail to validate workflow version:%s", version.ID) - } - workflowFiles := []string{mainWorkflowPath} - // need to start from line 2(start with line 0) - for i := 2; i < len(validateResultLines); i++ { - absPath := validateResultLines[i] - if len(absPath) == 0 { - continue - } - // validate file - if _, err := os.Stat(absPath); err == nil { - // in mac absPath was prefix with /private - relPath, err := filepath.Rel(baseDir, absPath[strings.LastIndex(absPath, baseDir):]) - if err != nil { - return apperrors.NewInternalError(err) - } - applog.Infow("file path", "baseDir", baseDir, "absPath", absPath, "relPath", relPath) - workflowFiles = append(workflowFiles, relPath) - } - } - for _, relPath := range workflowFiles { - input, err := os.ReadFile(path.Join(baseDir, relPath)) - if err != nil { - applog.Errorw("fail to read file", "err", err) - return apperrors.NewInternalError(err) - } - - encodedContent := base64.StdEncoding.EncodeToString(input) - - workflowFile, err := version.AddFile(&FileParam{ - Path: relPath, - Content: encodedContent, - }) + for _, value := range fileParams { + valueCopy := value + workflowFile, err := version.AddFile(&valueCopy) if err != nil { return err } applog.Infow("success add workflow file", "workflowVersionID", version.ID, "fileID", workflowFile.ID, "path", workflowFile.Path) } - return nil -} - -func (h *WorkflowVersionAddedHandler) getWorkflowInputs(ctx context.Context, WorkflowFilePath string) ([]WorkflowParam, error) { - return h.getWorkflowParams(ctx, "java", "-jar", h.womtoolPath, "inputs", WorkflowFilePath) -} -func (h *WorkflowVersionAddedHandler) getWorkflowOutputs(ctx context.Context, WorkflowFilePath string) ([]WorkflowParam, error) { - return h.getWorkflowParams(ctx, "java", "-jar", h.womtoolPath, "outputs", WorkflowFilePath) -} - -func (h *WorkflowVersionAddedHandler) getWorkflowParams(ctx context.Context, name string, arg ...string) ([]WorkflowParam, error) { - params := make([]WorkflowParam, 0) - outputsResult, err := exec.Exec(ctx, CommandExecuteTimeout, name, arg...) + // step4: get workflow inputs + inputsStr, err := workflowParser.GetWorkflowInputs(ctx, mainWorkflowPath) if err != nil { - return params, err - } - var outputsMap map[string]string - if err := json.Unmarshal(outputsResult, &outputsMap); err != nil { - return params, err - } - - for paramName, value := range outputsMap { - paramType, optional, defaultValue := parseWorkflowParamValue(value) - param := WorkflowParam{ - Name: paramName, - Type: paramType, - Optional: optional, - } - if defaultValue != nil { - param.Default = *defaultValue - } - params = append(params, param) + return err } - // keep the return sort stable - sort.Slice(params, func(i, j int) bool { - return params[i].Name < params[j].Name - }) - return params, nil -} - -func (h *WorkflowVersionAddedHandler) getWorkflowGraph(ctx context.Context, WorkflowFilePath string) (string, error) { - graph, err := exec.Exec(ctx, CommandExecuteTimeout, "java", "-jar", h.womtoolPath, "graph", WorkflowFilePath) + inputs, err := workflowParamPOToDO(inputsStr) if err != nil { - return "", err - } - - return string(graph), nil -} - -func parseWorkflowParamValue(value string) (paramType string, optional bool, defaultValue *string) { - splitByLeftBracket := strings.SplitN(value, "(", 2) - paramType = strings.TrimSpace(splitByLeftBracket[0]) - if len(splitByLeftBracket) == 1 { - return paramType, false, nil + return err } + version.Inputs = inputs - extraInfo := strings.TrimSuffix(splitByLeftBracket[1], ")") - splitByComma := strings.SplitN(extraInfo, ",", 2) - if strings.TrimSpace(splitByComma[0]) == "optional" { - optional = true + // step5: get workflow outputs + outputsStr, err := workflowParser.GetWorkflowOutputs(ctx, mainWorkflowPath) + if err != nil { + return err } - if len(splitByComma) == 1 { - return paramType, optional, nil + outputs, err := workflowParamPOToDO(outputsStr) + if err != nil { + return err } + version.Outputs = outputs - defaultInfo := strings.TrimSpace(splitByComma[1]) - splitByEqual := strings.SplitN(defaultInfo, "=", 2) - if len(splitByEqual) != 2 || strings.ToLower(strings.TrimSpace(splitByEqual[0])) != "default" { - return paramType, optional, nil - } - rawDefaultValue := strings.TrimSpace(splitByEqual[1]) - defaultValue = new(string) - if strings.HasPrefix(rawDefaultValue, `"`) && strings.HasSuffix(rawDefaultValue, `"`) { // String type - _ = json.Unmarshal([]byte(rawDefaultValue), defaultValue) // escape, never error - } else { - *defaultValue = rawDefaultValue + // step6: get workflow graph + graph, err := workflowParser.GetWorkflowGraph(ctx, mainWorkflowPath) + if err != nil { + return err } - return paramType, optional, defaultValue + version.Graph = graph + return nil } type WorkspaceDeletedHandler struct { diff --git a/internal/context/workspace/domain/workflow/object.go b/internal/context/workspace/domain/workflow/object.go index c0dd801..646fed6 100644 --- a/internal/context/workspace/domain/workflow/object.go +++ b/internal/context/workspace/domain/workflow/object.go @@ -1,6 +1,7 @@ package workflow import ( + "encoding/json" "time" applog "github.com/Bio-OS/bioos/pkg/log" @@ -141,6 +142,16 @@ func (v *WorkflowVersion) AddFile(param *FileParam) (*WorkflowFile, error) { return file, nil } +func workflowParamPOToDO(paramStr string) ([]WorkflowParam, error) { + var params []WorkflowParam + if len(paramStr) > 0 { + if err := json.Unmarshal([]byte(paramStr), ¶ms); err != nil { + return nil, err + } + } + return params, nil +} + // WorkflowFile workflow file type WorkflowFile struct { // ID is the unique identifier of the workflow file @@ -154,3 +165,13 @@ type WorkflowFile struct { // UpdatedAt is the update time of workflow file UpdatedAt time.Time } + +func fileParamPOToDO(paramStr string) ([]FileParam, error) { + var params []FileParam + if len(paramStr) > 0 { + if err := json.Unmarshal([]byte(paramStr), ¶ms); err != nil { + return nil, err + } + } + return params, nil +} diff --git a/internal/context/workspace/domain/workflow/service.go b/internal/context/workspace/domain/workflow/service.go index 8fbecc4..a9c5265 100644 --- a/internal/context/workspace/domain/workflow/service.go +++ b/internal/context/workspace/domain/workflow/service.go @@ -26,14 +26,14 @@ type service struct { factory *Factory } -func NewService(repo Repository, readModel workflow.ReadModel, bus eventbus.EventBus, factory *Factory, womtoolPath string) Service { +func NewService(repo Repository, readModel workflow.ReadModel, bus eventbus.EventBus, factory *Factory) Service { svc := &service{ readModel: readModel, repository: repo, eventbus: bus, factory: factory, } - svc.subscribeEvents(womtoolPath) + svc.subscribeEvents() return svc } @@ -200,7 +200,7 @@ func (s *service) UpdateVersion(ctx context.Context, workspaceID string, workflo return nil } -func (s *service) subscribeEvents(womtoolPath string) { +func (s *service) subscribeEvents() { s.eventbus.Subscribe(WorkflowVersionAdded, eventbus.EventHandlerFunc(func(ctx context.Context, payload string) (err error) { applog.Infow("start to consume workflow version added event", "payload", payload) @@ -209,7 +209,7 @@ func (s *service) subscribeEvents(womtoolPath string) { return err } - handler := NewWorkflowVersionAddedHandler(s.repository, womtoolPath) + handler := NewWorkflowVersionAddedHandler(s.repository) return handler.Handle(ctx, event) })) diff --git a/internal/context/workspace/infrastructure/workflowparser/object.go b/internal/context/workspace/infrastructure/workflowparser/object.go new file mode 100644 index 0000000..974a694 --- /dev/null +++ b/internal/context/workspace/infrastructure/workflowparser/object.go @@ -0,0 +1,65 @@ +package workflowparser + +import ( + "encoding/json" +) + +type WorkflowParam struct { + // Name param name + Name string `json:"name"` + // Type param type + Type string `json:"type"` + // Optional param is optional + Optional bool `json:"optional"` + // Default param default value + Default string `json:"default,omitempty"` +} + +func workflowParamPOToDO(paramStr string) ([]WorkflowParam, error) { + var params []WorkflowParam + if len(paramStr) > 0 { + if err := json.Unmarshal([]byte(paramStr), ¶ms); err != nil { + return nil, err + } + } + return params, nil +} + +func workflowParamDoToPO(params []WorkflowParam) (string, error) { + if len(params) == 0 { + return "", nil + } + + paramBytes, err := json.Marshal(params) + if err != nil { + return "", err + } + return string(paramBytes), nil +} + +type FileParam struct { + Path string `json:"path"` + Content string `json:"content"` +} + +func fileParamPOToDO(paramStr string) ([]FileParam, error) { + var params []FileParam + if len(paramStr) > 0 { + if err := json.Unmarshal([]byte(paramStr), ¶ms); err != nil { + return nil, err + } + } + return params, nil +} + +func fileParamDoToPO(params []FileParam) (string, error) { + if len(params) == 0 { + return "", nil + } + + paramBytes, err := json.Marshal(params) + if err != nil { + return "", err + } + return string(paramBytes), nil +} diff --git a/internal/context/workspace/infrastructure/workflowparser/wdlparser.go b/internal/context/workspace/infrastructure/workflowparser/wdlparser.go new file mode 100644 index 0000000..5b2392b --- /dev/null +++ b/internal/context/workspace/infrastructure/workflowparser/wdlparser.go @@ -0,0 +1,200 @@ +package workflowparser + +import ( + "bufio" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path" + "path/filepath" + "regexp" + "sort" + "strings" + "time" + + "github.com/Bio-OS/bioos/internal/context/workspace/interface/grpc/proto" + apperrors "github.com/Bio-OS/bioos/pkg/errors" + applog "github.com/Bio-OS/bioos/pkg/log" + "github.com/Bio-OS/bioos/pkg/utils/exec" +) + +type WDLConfig struct { + WomtoolPath string +} + +type WDLParser struct { + Config WDLConfig +} + +func NewWDLParser(config WDLConfig) *WDLParser { + return &WDLParser{Config: config} +} + +const ( + CommandExecuteTimeout = time.Minute * 3 + Language = "WDL" + VersionRegexpStr = "^version\\s+([\\w-._]+)" +) + +func (wdl *WDLParser) ParseWorkflowVersion(_ context.Context, mainWorkflowPath string) (string, error) { + versionRegexp := regexp.MustCompile(VersionRegexpStr) + file, err := os.Open(mainWorkflowPath) + if err != nil { + applog.Errorw("fail to open main workflow file", "err", err) + return "", err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + matched := versionRegexp.FindStringSubmatch(line) + if matched != nil && len(matched) >= 2 { + applog.Infow("version regexp matched", "matched", matched) + return matched[1], nil + } + } + + if err := scanner.Err(); err != nil { + return "", err + } + return "draft-2", nil +} + +func (wdl *WDLParser) ValidateWorkflowFiles(ctx context.Context, baseDir, mainWorkflowPath string) (string, error) { + applog.Infow("start to validate files", "mainWorkflowPath", mainWorkflowPath) + validateResult, err := exec.Exec(ctx, CommandExecuteTimeout, "java", "-jar", wdl.Config.WomtoolPath, "validate", path.Join(baseDir, mainWorkflowPath), "-l") + if err != nil { + applog.Errorw("fail to validate workflow", "err", err, "result", string(validateResult)) + return "", apperrors.NewInternalError(fmt.Errorf("validate workflow failed")) + } + validateResultLines := strings.Split(string(validateResult), "\n") + applog.Infow("validate result", "result", validateResultLines) + // parse and save workflow files + if len(validateResultLines) < 2 || strings.ToLower(validateResultLines[0]) != "success!" { + return "", proto.ErrorWorkflowValidateError("fail to validate workflow") + } + workflowFiles := []string{mainWorkflowPath} + // need to start from line 2(start with line 0) + for i := 2; i < len(validateResultLines); i++ { + absPath := validateResultLines[i] + if len(absPath) == 0 { + continue + } + // validate file + if _, err := os.Stat(absPath); err == nil { + // in mac absPath was prefix with /private + relPath, err := filepath.Rel(baseDir, absPath[strings.LastIndex(absPath, baseDir):]) + if err != nil { + return "", apperrors.NewInternalError(err) + } + applog.Infow("file path", "baseDir", baseDir, "absPath", absPath, "relPath", relPath) + workflowFiles = append(workflowFiles, relPath) + } + } + params := make([]FileParam, 0) + for _, relPath := range workflowFiles { + input, err := os.ReadFile(path.Join(baseDir, relPath)) + if err != nil { + applog.Errorw("fail to read file", "err", err) + return "", apperrors.NewInternalError(err) + } + + encodedContent := base64.StdEncoding.EncodeToString(input) + + param := FileParam{ + Path: relPath, + Content: encodedContent, + } + params = append(params, param) + } + return fileParamDoToPO(params) +} + +func (wdl *WDLParser) GetWorkflowInputs(ctx context.Context, WorkflowFilePath string) (string, error) { + results, err := wdl.GetWorkflowParams(ctx, "java", "-jar", wdl.Config.WomtoolPath, "inputs", WorkflowFilePath) + if err != nil { + return "", err + } + return workflowParamDoToPO(results) +} + +func (wdl *WDLParser) GetWorkflowOutputs(ctx context.Context, WorkflowFilePath string) (string, error) { + results, err := wdl.GetWorkflowParams(ctx, "java", "-jar", wdl.Config.WomtoolPath, "outputs", WorkflowFilePath) + if err != nil { + return "", err + } + return workflowParamDoToPO(results) +} + +func (wdl *WDLParser) GetWorkflowParams(ctx context.Context, name string, arg ...string) ([]WorkflowParam, error) { + params := make([]WorkflowParam, 0) + outputsResult, err := exec.Exec(ctx, CommandExecuteTimeout, name, arg...) + if err != nil { + return params, err + } + var outputsMap map[string]string + if err := json.Unmarshal(outputsResult, &outputsMap); err != nil { + return params, err + } + + for paramName, value := range outputsMap { + paramType, optional, defaultValue := parseWorkflowParamValue(value) + param := WorkflowParam{ + Name: paramName, + Type: paramType, + Optional: optional, + } + if defaultValue != nil { + param.Default = *defaultValue + } + params = append(params, param) + } + // keep the return sort stable + sort.Slice(params, func(i, j int) bool { + return params[i].Name < params[j].Name + }) + return params, nil +} + +func (wdl *WDLParser) GetWorkflowGraph(ctx context.Context, WorkflowFilePath string) (string, error) { + graph, err := exec.Exec(ctx, CommandExecuteTimeout, "java", "-jar", wdl.Config.WomtoolPath, "graph", WorkflowFilePath) + if err != nil { + return "", err + } + + return string(graph), nil +} + +func parseWorkflowParamValue(value string) (paramType string, optional bool, defaultValue *string) { + splitByLeftBracket := strings.SplitN(value, "(", 2) + paramType = strings.TrimSpace(splitByLeftBracket[0]) + if len(splitByLeftBracket) == 1 { + return paramType, false, nil + } + + extraInfo := strings.TrimSuffix(splitByLeftBracket[1], ")") + splitByComma := strings.SplitN(extraInfo, ",", 2) + if strings.TrimSpace(splitByComma[0]) == "optional" { + optional = true + } + if len(splitByComma) == 1 { + return paramType, optional, nil + } + + defaultInfo := strings.TrimSpace(splitByComma[1]) + splitByEqual := strings.SplitN(defaultInfo, "=", 2) + if len(splitByEqual) != 2 || strings.ToLower(strings.TrimSpace(splitByEqual[0])) != "default" { + return paramType, optional, nil + } + rawDefaultValue := strings.TrimSpace(splitByEqual[1]) + defaultValue = new(string) + if strings.HasPrefix(rawDefaultValue, `"`) && strings.HasSuffix(rawDefaultValue, `"`) { // String type + _ = json.Unmarshal([]byte(rawDefaultValue), defaultValue) // escape, never error + } else { + *defaultValue = rawDefaultValue + } + return paramType, optional, defaultValue +} diff --git a/internal/context/workspace/infrastructure/workflowparser/workflowparser.go b/internal/context/workspace/infrastructure/workflowparser/workflowparser.go new file mode 100644 index 0000000..9b701ff --- /dev/null +++ b/internal/context/workspace/infrastructure/workflowparser/workflowparser.go @@ -0,0 +1,77 @@ +package workflowparser + +import ( + "context" + "fmt" + "sync" +) + +const ( + WDL = "WDL" + CWL = "CWL" + Snakemake = "SMK" + Nextflow = "NFL" +) + +type ParserConfig interface{} + +type WorkflowParser interface { + ParseWorkflowVersion(ctx context.Context, mainWorkflowPath string) (string, error) + ValidateWorkflowFiles(ctx context.Context, baseDir, mainWorkflowPath string) (string, error) + GetWorkflowInputs(ctx context.Context, WorkflowFilePath string) (string, error) + GetWorkflowOutputs(ctx context.Context, WorkflowFilePath string) (string, error) + GetWorkflowGraph(ctx context.Context, WorkflowFilePath string) (string, error) +} + +var parserCreateFunc = make(map[string]func(ParserConfig) WorkflowParser) + +func RegisterParseCreateFunc(workflowType string, function func(ParserConfig) WorkflowParser) { + parserCreateFunc[workflowType] = function +} + +func init() { + RegisterParseCreateFunc(WDL, func(config ParserConfig) WorkflowParser { + wdlConfig, ok := config.(WDLConfig) + if !ok { + panic("Invalid config type for WDL parser") + } + return NewWDLParser(wdlConfig) + }) +} + +var ( + instance *WorkflowParserFactory + once sync.Once +) + +type WorkflowParserFactory struct { + parsers map[string]WorkflowParser +} + +func InitWorkflowParserFactory(configs map[string]ParserConfig) { + once.Do(func() { + instance = &WorkflowParserFactory{ + parsers: make(map[string]WorkflowParser), + } + + for workflowType, config := range configs { + createFunc, exists := parserCreateFunc[workflowType] + if !exists { + panic("no parser registered for workflow type") + } + instance.parsers[workflowType] = createFunc(config) + } + }) +} + +func GetFactory() *WorkflowParserFactory { + return instance +} + +func (f *WorkflowParserFactory) GetParser(workflowType string) (WorkflowParser, error) { + parser, exists := f.parsers[workflowType] + if !exists { + return nil, fmt.Errorf("no parser found for workflow type: %s", workflowType) + } + return parser, nil +} From 4c1eade8e6e00f7a2e58edb4773e6f0f21ace3d2 Mon Sep 17 00:00:00 2001 From: Bing64 <625255264@qq.com> Date: Sun, 3 Dec 2023 02:20:33 +0800 Subject: [PATCH 03/18] Initial support for CWL language import --- .../application/command/workflow/commands.go | 3 + .../workflowparser/cwlparser.go | 206 ++++++++++++++++++ .../infrastructure/workflowparser/object.go | 7 + .../workflowparser/wdlparser.go | 9 +- .../workflowparser/workflowparser.go | 11 +- 5 files changed, 226 insertions(+), 10 deletions(-) create mode 100644 internal/context/workspace/infrastructure/workflowparser/cwlparser.go diff --git a/internal/context/workspace/application/command/workflow/commands.go b/internal/context/workspace/application/command/workflow/commands.go index 7b5cfcd..e5a4fcb 100644 --- a/internal/context/workspace/application/command/workflow/commands.go +++ b/internal/context/workspace/application/command/workflow/commands.go @@ -20,6 +20,9 @@ func NewCommands(repo workflow.Repository, workflowReadModel workflowquery.ReadM workflowparser.WDL: workflowparser.WDLConfig{ WomtoolPath: womtoolPath, }, + workflowparser.CWL: workflowparser.CWLConfig{ + CwltoolCmd: "cwltool", + }, } workflowparser.InitWorkflowParserFactory(configs) return &Commands{ diff --git a/internal/context/workspace/infrastructure/workflowparser/cwlparser.go b/internal/context/workspace/infrastructure/workflowparser/cwlparser.go new file mode 100644 index 0000000..aa597f3 --- /dev/null +++ b/internal/context/workspace/infrastructure/workflowparser/cwlparser.go @@ -0,0 +1,206 @@ +package workflowparser + +import ( + "bufio" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path" + "regexp" + "strings" + + "github.com/Bio-OS/bioos/internal/context/workspace/interface/grpc/proto" + apperrors "github.com/Bio-OS/bioos/pkg/errors" + applog "github.com/Bio-OS/bioos/pkg/log" + "github.com/Bio-OS/bioos/pkg/utils/exec" + "gopkg.in/yaml.v2" +) + +type CWLConfig struct { + CwltoolCmd string +} + +type CWLParser struct { + Config CWLConfig +} + +func NewCWLParser(config CWLConfig) *CWLParser { + return &CWLParser{Config: config} +} + +func (cwl *CWLParser) ParseWorkflowVersion(_ context.Context, mainWorkflowPath string) (string, error) { + versionRegexp := regexp.MustCompile(CWLVersionRegexpStr) + file, err := os.Open(mainWorkflowPath) + if err != nil { + applog.Errorw("fail to open main workflow file", "err", err) + return "", err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + matched := versionRegexp.FindStringSubmatch(line) + if len(matched) >= 2 { + applog.Infow("version regexp matched", "matched", matched) + return matched[1], nil + } + } + + if err := scanner.Err(); err != nil { + return "", err + } + return "v1.0", nil +} + +func (cwl *CWLParser) ValidateWorkflowFiles(ctx context.Context, baseDir, mainWorkflowPath string) (string, error) { + applog.Infow("start to validate files", "mainWorkflowPath", mainWorkflowPath) + validateResult, err := exec.Exec(ctx, CommandExecuteTimeout, cwl.Config.CwltoolCmd, "--validate", path.Join(baseDir, mainWorkflowPath)) + if err != nil { + applog.Errorw("fail to validate workflow", "err", err, "result", string(validateResult)) + return "", apperrors.NewInternalError(fmt.Errorf("validate workflow failed")) + } + validateResultLines := strings.Split(string(validateResult), "\n") + applog.Infow("validate result", "result", validateResultLines) + // parse and save workflow files + if len(validateResultLines) < 3 || !strings.Contains(validateResultLines[2], "is valid CWL") { + return "", proto.ErrorWorkflowValidateError("fail to validate workflow") + } + + depsResults, err := exec.Exec(ctx, CommandExecuteTimeout, cwl.Config.CwltoolCmd, "--print-deps", path.Join(baseDir, mainWorkflowPath)) + if err != nil { + return "", err + } + depsResultLines := strings.Split(string(depsResults), "\n") + if len(validateResultLines) < 3 { + return "", proto.ErrorWorkflowValidateError("fail to check deps") + } + jsonContent := strings.Join(depsResultLines[2:], "\n") + var file CWLFile + err = json.Unmarshal([]byte(jsonContent), &file) + if err != nil { + return "", err + } + + workflowFiles := []string{mainWorkflowPath} + // need to start from line 2(start with line 0) + for i := 0; i < len(file.SecondaryFiles); i++ { + workflowFiles = append(workflowFiles, file.SecondaryFiles[i].Location) + } + params := make([]FileParam, 0) + for _, relPath := range workflowFiles { + input, err := os.ReadFile(path.Join(baseDir, relPath)) + if err != nil { + applog.Errorw("fail to read file", "err", err) + return "", apperrors.NewInternalError(err) + } + + encodedContent := base64.StdEncoding.EncodeToString(input) + + param := FileParam{ + Path: relPath, + Content: encodedContent, + } + params = append(params, param) + } + return fileParamDoToPO(params) +} + +func (cwl *CWLParser) GetWorkflowInputs(ctx context.Context, WorkflowFilePath string) (string, error) { + params := make([]WorkflowParam, 0) + templateResult, err := exec.Exec(ctx, CommandExecuteTimeout, cwl.Config.CwltoolCmd, "--make-template", WorkflowFilePath) + if err != nil { + return "", err + } + templateResultLines := strings.Split(string(templateResult), "\n") + yamlContent := strings.Join(templateResultLines[2:], "\n") + + var yamlData map[string]interface{} + if err := yaml.Unmarshal([]byte(yamlContent), &yamlData); err != nil { + return "", err + } + + for key, value := range yamlData { + param := WorkflowParam{Name: key} + + keyWithColon := key + ":" + startIndex := strings.Index(yamlContent, keyWithColon) + if startIndex == -1 { + continue + } + endIndex := strings.Index(yamlContent[startIndex:], "\n") + var paramDesc string + if endIndex == -1 { + paramDesc = yamlContent[startIndex:] + } else { + paramDesc = yamlContent[startIndex:endIndex] + } + + reType := regexp.MustCompile(`type\s+'([^']*)'`) + matchType := reType.FindStringSubmatch(paramDesc) + if matchType != nil { + param.Type = matchType[1] + } else { + continue + } + + param.Optional = strings.Contains(paramDesc, "(optional)") + + if strings.Contains(paramDesc, "default value") { + param.Default = value.(string) + } + + params = append(params, param) + } + + return workflowParamDoToPO(params) +} + +func (cwl *CWLParser) GetWorkflowOutputs(ctx context.Context, WorkflowFilePath string) (string, error) { + params := make([]WorkflowParam, 0) + rdfResult, err := exec.Exec(ctx, CommandExecuteTimeout, cwl.Config.CwltoolCmd, "--print-rdf", WorkflowFilePath) + if err != nil { + return "", err + } + rdfResultSection := strings.Split(string(rdfResult), "\n\n") + for _, value := range rdfResultSection { + if strings.Contains(value, "cwl:outputBinding") { + param := WorkflowParam{Optional: false} + re := regexp.MustCompile(`sld:type\s+([^:\s]+):`) + match := re.FindStringSubmatch(value) + if len(match) > 1 { + param.Type = match[1] + } else { + continue + } + + re2 := regexp.MustCompile(`\/(.*?)> rdfs:comment`) + match2 := re2.FindStringSubmatch(value) + if len(match2) > 1 { + param.Name = match[1] + } else { + continue + } + param.Default = value + params = append(params, param) + } + } + + return workflowParamDoToPO(params) +} + +func (cwl *CWLParser) GetWorkflowGraph(ctx context.Context, WorkflowFilePath string) (string, error) { + return "", nil +} + +type CWLFile struct { + Class string `json:"class"` + Location string `json:"location"` + Format string `json:"format"` + SecondaryFiles []CWLFile `json:"secondaryFiles,omitempty"` // 使用相同的结构体表示嵌套的secondaryFiles数组 + Basename string `json:"basename,omitempty"` // omitempty保证如果字段为空，则在JSON中省略 + Nameroot string `json:"nameroot,omitempty"` + Nameext string `json:"nameext,omitempty"` +} diff --git a/internal/context/workspace/infrastructure/workflowparser/object.go b/internal/context/workspace/infrastructure/workflowparser/object.go index 974a694..e36239a 100644 --- a/internal/context/workspace/infrastructure/workflowparser/object.go +++ b/internal/context/workspace/infrastructure/workflowparser/object.go @@ -2,6 +2,13 @@ package workflowparser import ( "encoding/json" + "time" +) + +const ( + CommandExecuteTimeout = time.Minute * 3 + WDLVersionRegexpStr = "^version\\s+([\\w-._]+)" + CWLVersionRegexpStr = "^cwlVersion\\s+([\\w-._]+)" ) type WorkflowParam struct { diff --git a/internal/context/workspace/infrastructure/workflowparser/wdlparser.go b/internal/context/workspace/infrastructure/workflowparser/wdlparser.go index 5b2392b..d1fe9dd 100644 --- a/internal/context/workspace/infrastructure/workflowparser/wdlparser.go +++ b/internal/context/workspace/infrastructure/workflowparser/wdlparser.go @@ -12,7 +12,6 @@ import ( "regexp" "sort" "strings" - "time" "github.com/Bio-OS/bioos/internal/context/workspace/interface/grpc/proto" apperrors "github.com/Bio-OS/bioos/pkg/errors" @@ -32,14 +31,8 @@ func NewWDLParser(config WDLConfig) *WDLParser { return &WDLParser{Config: config} } -const ( - CommandExecuteTimeout = time.Minute * 3 - Language = "WDL" - VersionRegexpStr = "^version\\s+([\\w-._]+)" -) - func (wdl *WDLParser) ParseWorkflowVersion(_ context.Context, mainWorkflowPath string) (string, error) { - versionRegexp := regexp.MustCompile(VersionRegexpStr) + versionRegexp := regexp.MustCompile(WDLVersionRegexpStr) file, err := os.Open(mainWorkflowPath) if err != nil { applog.Errorw("fail to open main workflow file", "err", err) diff --git a/internal/context/workspace/infrastructure/workflowparser/workflowparser.go b/internal/context/workspace/infrastructure/workflowparser/workflowparser.go index 9b701ff..60b9938 100644 --- a/internal/context/workspace/infrastructure/workflowparser/workflowparser.go +++ b/internal/context/workspace/infrastructure/workflowparser/workflowparser.go @@ -31,11 +31,18 @@ func RegisterParseCreateFunc(workflowType string, function func(ParserConfig) Wo func init() { RegisterParseCreateFunc(WDL, func(config ParserConfig) WorkflowParser { - wdlConfig, ok := config.(WDLConfig) + configParam, ok := config.(WDLConfig) if !ok { panic("Invalid config type for WDL parser") } - return NewWDLParser(wdlConfig) + return NewWDLParser(configParam) + }) + RegisterParseCreateFunc(CWL, func(config ParserConfig) WorkflowParser { + configParam, ok := config.(CWLConfig) + if !ok { + panic("Invalid config type for WDL parser") + } + return NewCWLParser(configParam) }) } From c9f922bf900e59b362250643bab3d00f71186b1f Mon Sep 17 00:00:00 2001 From: Bing64 <625255264@qq.com> Date: Sun, 3 Dec 2023 12:43:25 +0800 Subject: [PATCH 04/18] Fix for CWL parser --- .../workflowparser/cwlparser.go | 170 +++++++++++++----- 1 file changed, 123 insertions(+), 47 deletions(-) diff --git a/internal/context/workspace/infrastructure/workflowparser/cwlparser.go b/internal/context/workspace/infrastructure/workflowparser/cwlparser.go index aa597f3..d37f339 100644 --- a/internal/context/workspace/infrastructure/workflowparser/cwlparser.go +++ b/internal/context/workspace/infrastructure/workflowparser/cwlparser.go @@ -10,12 +10,13 @@ import ( "path" "regexp" "strings" + "unicode" + "unicode/utf8" "github.com/Bio-OS/bioos/internal/context/workspace/interface/grpc/proto" apperrors "github.com/Bio-OS/bioos/pkg/errors" applog "github.com/Bio-OS/bioos/pkg/log" "github.com/Bio-OS/bioos/pkg/utils/exec" - "gopkg.in/yaml.v2" ) type CWLConfig struct { @@ -74,7 +75,7 @@ func (cwl *CWLParser) ValidateWorkflowFiles(ctx context.Context, baseDir, mainWo return "", err } depsResultLines := strings.Split(string(depsResults), "\n") - if len(validateResultLines) < 3 { + if len(depsResultLines) < 3 { return "", proto.ErrorWorkflowValidateError("fail to check deps") } jsonContent := strings.Join(depsResultLines[2:], "\n") @@ -110,49 +111,69 @@ func (cwl *CWLParser) ValidateWorkflowFiles(ctx context.Context, baseDir, mainWo func (cwl *CWLParser) GetWorkflowInputs(ctx context.Context, WorkflowFilePath string) (string, error) { params := make([]WorkflowParam, 0) - templateResult, err := exec.Exec(ctx, CommandExecuteTimeout, cwl.Config.CwltoolCmd, "--make-template", WorkflowFilePath) + rdfResult, err := exec.Exec(ctx, CommandExecuteTimeout, cwl.Config.CwltoolCmd, "--print-rdf", WorkflowFilePath) if err != nil { return "", err } - templateResultLines := strings.Split(string(templateResult), "\n") - yamlContent := strings.Join(templateResultLines[2:], "\n") - var yamlData map[string]interface{} - if err := yaml.Unmarshal([]byte(yamlContent), &yamlData); err != nil { - return "", err + inputsRe := regexp.MustCompile(`(?s)cwl:inputs\s(.*?)\s;`) // 找到所有输入文件 + inputs := inputsRe.FindAllStringSubmatch(string(rdfResult), -1) + var inputFiles []string + if len(inputs) > 0 { + for _, value := range inputs { + files := strings.Split(value[1], ",") + for _, file := range files { + trimmedFile := strings.TrimSpace(file) + inputFiles = append(inputFiles, trimmedFile) + } + } + } else { + return "", nil } - for key, value := range yamlData { - param := WorkflowParam{Name: key} + sectionRe := regexp.MustCompile(`[\s]*\r?\n[\s]*\r?\n[\s]*`) // 输出结果分块 + rdfResultSection := sectionRe.Split(string(rdfResult), -1) - keyWithColon := key + ":" - startIndex := strings.Index(yamlContent, keyWithColon) - if startIndex == -1 { - continue - } - endIndex := strings.Index(yamlContent[startIndex:], "\n") - var paramDesc string - if endIndex == -1 { - paramDesc = yamlContent[startIndex:] - } else { - paramDesc = yamlContent[startIndex:endIndex] - } + for _, value := range rdfResultSection { // 遍历各个分块，找到描述对应文件的部分 + if containsAny(value, inputFiles) && !strings.Contains(value, "cwl:inputs") { + param := WorkflowParam{} + nameRe := regexp.MustCompile(`#(.*?)> `) + nameMatch := nameRe.FindStringSubmatch(value) + if len(nameMatch) > 1 { + param.Name = nameMatch[1] + lastSlashIndex := strings.LastIndex(param.Name, "/") // 某些情况下会多一级斜杠 + if lastSlashIndex != -1 { + param.Name = param.Name[lastSlashIndex+1:] + } + } else { + continue + } - reType := regexp.MustCompile(`type\s+'([^']*)'`) - matchType := reType.FindStringSubmatch(paramDesc) - if matchType != nil { - param.Type = matchType[1] - } else { - continue - } + param.Optional = strings.Contains(value, "sld:null") - param.Optional = strings.Contains(paramDesc, "(optional)") + typeRe := regexp.MustCompile(`sld:type\s+.*?([^:\s,.]+)[\s,.]`) + typeMatch := typeRe.FindAllStringSubmatch(value, -1) + if len(typeMatch) > 0 { + if strings.Contains(typeMatch[0][1], "[") { // Array Enum Record等类型，文本格式与其他类型不同 + param.Type = upperCaseFirstLetter(typeMatch[len(typeMatch)-1][1]) + } else { + param.Type = upperCaseFirstLetter(typeMatch[0][1]) + } + } else { + continue + } - if strings.Contains(paramDesc, "default value") { - param.Default = value.(string) + defaultRe := regexp.MustCompile(`cwl:default\s+.*?([^;]+)\s`) + defaultMatch := defaultRe.FindStringSubmatch(value) + if len(defaultMatch) > 1 && param.Type != "Record" { // Record类型暂时无法输出默认值 + if param.Type == "Array" { + fmt.Println("[" + removeSpacesAndNewlines(defaultMatch[1]) + "]") + } else { + param.Default = defaultMatch[1] + } + } + params = append(params, param) } - - params = append(params, param) } return workflowParamDoToPO(params) @@ -164,26 +185,56 @@ func (cwl *CWLParser) GetWorkflowOutputs(ctx context.Context, WorkflowFilePath s if err != nil { return "", err } - rdfResultSection := strings.Split(string(rdfResult), "\n\n") - for _, value := range rdfResultSection { - if strings.Contains(value, "cwl:outputBinding") { - param := WorkflowParam{Optional: false} - re := regexp.MustCompile(`sld:type\s+([^:\s]+):`) - match := re.FindStringSubmatch(value) - if len(match) > 1 { - param.Type = match[1] + + outputsRe := regexp.MustCompile(`(?s)cwl:outputs\s(.*?)\s;`) // 找到所有输出文件 + outputs := outputsRe.FindAllStringSubmatch(string(rdfResult), -1) + var outputFiles []string + if len(outputs) > 0 { + for _, value := range outputs { + files := strings.Split(value[1], ",") + for _, file := range files { + trimmedFile := strings.TrimSpace(file) + outputFiles = append(outputFiles, trimmedFile) + } + } + } else { + return "", nil + } + + sectionRe := regexp.MustCompile(`[\s]*\r?\n[\s]*\r?\n[\s]*`) // 输出结果分块 + rdfResultSection := sectionRe.Split(string(rdfResult), -1) + + for _, value := range rdfResultSection { // 遍历各个分块，找到描述对应文件的部分 + if containsAny(value, outputFiles) && !strings.Contains(value, "cwl:outputs") { + param := WorkflowParam{} + nameRe := regexp.MustCompile(`#(.*?)> `) + nameMatch := nameRe.FindStringSubmatch(value) + if len(nameMatch) > 1 { + param.Name = nameMatch[1] + lastSlashIndex := strings.LastIndex(param.Name, "/") // 某些情况下会多一级斜杠 + if lastSlashIndex != -1 { + param.Name = param.Name[lastSlashIndex+1:] + } } else { continue } - re2 := regexp.MustCompile(`\/(.*?)> rdfs:comment`) - match2 := re2.FindStringSubmatch(value) - if len(match2) > 1 { - param.Name = match[1] + param.Optional = true // 输出全部为Optional + + typeRe := regexp.MustCompile(`sld:type\s+.*?([^:\s,.]+)[\s,.]`) + typeMatch := typeRe.FindAllStringSubmatch(value, -1) + if len(typeMatch) > 0 { + if strings.Contains(typeMatch[0][1], "[") { // Array Enum Record等类型，文本格式与其他类型不同 + param.Type = upperCaseFirstLetter(typeMatch[len(typeMatch)-1][1]) + } else { + param.Type = upperCaseFirstLetter(typeMatch[0][1]) + } } else { continue } - param.Default = value + + // 不处理default + params = append(params, param) } } @@ -204,3 +255,28 @@ type CWLFile struct { Nameroot string `json:"nameroot,omitempty"` Nameext string `json:"nameext,omitempty"` } + +func containsAny(str string, substrings []string) bool { + for _, substr := range substrings { + if strings.Contains(str, substr) { + return true + } + } + return false +} + +func upperCaseFirstLetter(s string) string { + if s == "" { + return "" + } + r, size := utf8.DecodeRuneInString(s) + return string(unicode.ToUpper(r)) + s[size:] +} + +func removeSpacesAndNewlines(input string) string { + parts := strings.Split(input, ",") + for i, part := range parts { + parts[i] = strings.TrimSpace(part) + } + return strings.Join(parts, ",") +} From d6c24662a702a88156c18fa7cc5ab2512b574101 Mon Sep 17 00:00:00 2001 From: Bing64 <625255264@qq.com> Date: Sun, 3 Dec 2023 18:00:32 +0800 Subject: [PATCH 05/18] Fix for CWL parser, support complex workflow --- .../workflowparser/cwlparser.go | 101 ++++++++++++------ 1 file changed, 69 insertions(+), 32 deletions(-) diff --git a/internal/context/workspace/infrastructure/workflowparser/cwlparser.go b/internal/context/workspace/infrastructure/workflowparser/cwlparser.go index d37f339..2a16e35 100644 --- a/internal/context/workspace/infrastructure/workflowparser/cwlparser.go +++ b/internal/context/workspace/infrastructure/workflowparser/cwlparser.go @@ -4,10 +4,10 @@ import ( "bufio" "context" "encoding/base64" - "encoding/json" "fmt" "os" "path" + "path/filepath" "regexp" "strings" "unicode" @@ -66,7 +66,7 @@ func (cwl *CWLParser) ValidateWorkflowFiles(ctx context.Context, baseDir, mainWo validateResultLines := strings.Split(string(validateResult), "\n") applog.Infow("validate result", "result", validateResultLines) // parse and save workflow files - if len(validateResultLines) < 3 || !strings.Contains(validateResultLines[2], "is valid CWL") { + if len(validateResultLines) < 3 || !strings.Contains(validateResultLines[len(validateResultLines)-2], "is valid CWL") { return "", proto.ErrorWorkflowValidateError("fail to validate workflow") } @@ -74,25 +74,43 @@ func (cwl *CWLParser) ValidateWorkflowFiles(ctx context.Context, baseDir, mainWo if err != nil { return "", err } - depsResultLines := strings.Split(string(depsResults), "\n") - if len(depsResultLines) < 3 { - return "", proto.ErrorWorkflowValidateError("fail to check deps") - } - jsonContent := strings.Join(depsResultLines[2:], "\n") - var file CWLFile - err = json.Unmarshal([]byte(jsonContent), &file) - if err != nil { + // depsResultLines := strings.Split(string(depsResults), "\n") + // if len(depsResultLines) < 3 { + // return "", proto.ErrorWorkflowValidateError("fail to check deps") + // } + // jsonContent := strings.Join(depsResultLines[2:], "\n") + // var file CWLFile + // err = json.Unmarshal([]byte(jsonContent), &file) + // if err != nil { + // return "", err + // } + depsFileRe := regexp.MustCompile(`"location":\s*"([^"]*)"`) // 结构比较复杂，没必要解析，直接匹配字符即可 + depsFiles := depsFileRe.FindAllStringSubmatch(string(depsResults), -1) + + workflowFiles := []string{} + if len(depsFiles) > 1 { + scriptDir := filepath.Dir(mainWorkflowPath) + for _, value := range depsFiles { + joinedPath := filepath.Join(scriptDir, value[1]) + cleanedPath := filepath.Clean(joinedPath) + if len(cleanedPath) == 0 { + continue + } + // validate file + if _, err := os.Stat(filepath.Join(baseDir, cleanedPath)); err == nil { + applog.Infow("file path", "baseDir", baseDir, "relPath", cleanedPath) + if !sliceContains(workflowFiles, cleanedPath) { // 去掉重复的文件 + workflowFiles = append(workflowFiles, cleanedPath) + } + } + } + } else { return "", err } - workflowFiles := []string{mainWorkflowPath} - // need to start from line 2(start with line 0) - for i := 0; i < len(file.SecondaryFiles); i++ { - workflowFiles = append(workflowFiles, file.SecondaryFiles[i].Location) - } params := make([]FileParam, 0) for _, relPath := range workflowFiles { - input, err := os.ReadFile(path.Join(baseDir, relPath)) + input, err := os.ReadFile(filepath.Join(baseDir, relPath)) if err != nil { applog.Errorw("fail to read file", "err", err) return "", apperrors.NewInternalError(err) @@ -120,12 +138,17 @@ func (cwl *CWLParser) GetWorkflowInputs(ctx context.Context, WorkflowFilePath st inputs := inputsRe.FindAllStringSubmatch(string(rdfResult), -1) var inputFiles []string if len(inputs) > 0 { - for _, value := range inputs { - files := strings.Split(value[1], ",") - for _, file := range files { - trimmedFile := strings.TrimSpace(file) - inputFiles = append(inputFiles, trimmedFile) - } + // for _, value := range inputs { + // files := strings.Split(value[1], ",") + // for _, file := range files { + // trimmedFile := strings.TrimSpace(file) + // inputFiles = append(inputFiles, trimmedFile) + // } + // } + files := strings.Split(inputs[0][1], ",") // 只处理第一部分，不需要中间文件 + for _, file := range files { + trimmedFile := strings.TrimSpace(file) + inputFiles = append(inputFiles, trimmedFile) } } else { return "", nil @@ -135,7 +158,7 @@ func (cwl *CWLParser) GetWorkflowInputs(ctx context.Context, WorkflowFilePath st rdfResultSection := sectionRe.Split(string(rdfResult), -1) for _, value := range rdfResultSection { // 遍历各个分块，找到描述对应文件的部分 - if containsAny(value, inputFiles) && !strings.Contains(value, "cwl:inputs") { + if findFileSection(value, inputFiles) && !strings.Contains(value, "cwl:inputs") { param := WorkflowParam{} nameRe := regexp.MustCompile(`#(.*?)> `) nameMatch := nameRe.FindStringSubmatch(value) @@ -190,12 +213,17 @@ func (cwl *CWLParser) GetWorkflowOutputs(ctx context.Context, WorkflowFilePath s outputs := outputsRe.FindAllStringSubmatch(string(rdfResult), -1) var outputFiles []string if len(outputs) > 0 { - for _, value := range outputs { - files := strings.Split(value[1], ",") - for _, file := range files { - trimmedFile := strings.TrimSpace(file) - outputFiles = append(outputFiles, trimmedFile) - } + // for _, value := range outputs { + // files := strings.Split(value[1], ",") + // for _, file := range files { + // trimmedFile := strings.TrimSpace(file) + // outputFiles = append(outputFiles, trimmedFile) + // } + // } + files := strings.Split(outputs[0][1], ",") // 只处理第一部分，不需要中间文件 + for _, file := range files { + trimmedFile := strings.TrimSpace(file) + outputFiles = append(outputFiles, trimmedFile) } } else { return "", nil @@ -205,7 +233,7 @@ func (cwl *CWLParser) GetWorkflowOutputs(ctx context.Context, WorkflowFilePath s rdfResultSection := sectionRe.Split(string(rdfResult), -1) for _, value := range rdfResultSection { // 遍历各个分块，找到描述对应文件的部分 - if containsAny(value, outputFiles) && !strings.Contains(value, "cwl:outputs") { + if findFileSection(value, outputFiles) && !strings.Contains(value, "cwl:outputs") { param := WorkflowParam{} nameRe := regexp.MustCompile(`#(.*?)> `) nameMatch := nameRe.FindStringSubmatch(value) @@ -256,9 +284,9 @@ type CWLFile struct { Nameext string `json:"nameext,omitempty"` } -func containsAny(str string, substrings []string) bool { +func findFileSection(str string, substrings []string) bool { for _, substr := range substrings { - if strings.Contains(str, substr) { + if strings.HasPrefix(str, substr) { return true } } @@ -280,3 +308,12 @@ func removeSpacesAndNewlines(input string) string { } return strings.Join(parts, ",") } + +func sliceContains(slice []string, element string) bool { + for _, v := range slice { + if v == element { + return true + } + } + return false +} From b4ec42519a1501be8ef5f39162dca734339fe3d1 Mon Sep 17 00:00:00 2001 From: Bing64 <625255264@qq.com> Date: Sun, 3 Dec 2023 18:52:24 +0800 Subject: [PATCH 06/18] Add GetWorkflowGraph for CWL parser --- .../infrastructure/workflowparser/cwlparser.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/internal/context/workspace/infrastructure/workflowparser/cwlparser.go b/internal/context/workspace/infrastructure/workflowparser/cwlparser.go index 2a16e35..283000f 100644 --- a/internal/context/workspace/infrastructure/workflowparser/cwlparser.go +++ b/internal/context/workspace/infrastructure/workflowparser/cwlparser.go @@ -271,9 +271,25 @@ func (cwl *CWLParser) GetWorkflowOutputs(ctx context.Context, WorkflowFilePath s } func (cwl *CWLParser) GetWorkflowGraph(ctx context.Context, WorkflowFilePath string) (string, error) { - return "", nil + graph, err := exec.Exec(ctx, CommandExecuteTimeout, cwl.Config.CwltoolCmd, "--print-dot", WorkflowFilePath) + if err != nil { // 无法绘图时不返回错误，使流程能成功导入 + return noGraph, nil + } + + graphRe := regexp.MustCompile(`(?s)(digraph.*\})`) + graphMatch := graphRe.FindStringSubmatch(string(graph)) + + if len(graphMatch) > 1 { + return graphMatch[1], nil + } else { + return noGraph, nil + } } +const noGraph = `digraph G { + "error" [shape=box, style=filled, color=lightgrey, label="The graph for this workflow is unavailable."]; +}` + type CWLFile struct { Class string `json:"class"` Location string `json:"location"` From 9075f3f75bb86686130e6a411bff5a04abd86367 Mon Sep 17 00:00:00 2001 From: Bing64 <625255264@qq.com> Date: Sun, 3 Dec 2023 19:13:54 +0800 Subject: [PATCH 07/18] Fixed a bug when parsing a single file workflow. --- .../workspace/infrastructure/workflowparser/cwlparser.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/context/workspace/infrastructure/workflowparser/cwlparser.go b/internal/context/workspace/infrastructure/workflowparser/cwlparser.go index 283000f..69bfc69 100644 --- a/internal/context/workspace/infrastructure/workflowparser/cwlparser.go +++ b/internal/context/workspace/infrastructure/workflowparser/cwlparser.go @@ -88,7 +88,7 @@ func (cwl *CWLParser) ValidateWorkflowFiles(ctx context.Context, baseDir, mainWo depsFiles := depsFileRe.FindAllStringSubmatch(string(depsResults), -1) workflowFiles := []string{} - if len(depsFiles) > 1 { + if len(depsFiles) > 0 { scriptDir := filepath.Dir(mainWorkflowPath) for _, value := range depsFiles { joinedPath := filepath.Join(scriptDir, value[1]) @@ -105,7 +105,7 @@ func (cwl *CWLParser) ValidateWorkflowFiles(ctx context.Context, baseDir, mainWo } } } else { - return "", err + return "", apperrors.NewInternalError(fmt.Errorf("no valid workflow files")) } params := make([]FileParam, 0) From e4fa6de90e2889945762b61421921b23a9b94ff5 Mon Sep 17 00:00:00 2001 From: Bing64 <625255264@qq.com> Date: Mon, 4 Dec 2023 00:25:26 +0800 Subject: [PATCH 08/18] Add a language selector to bioos web --- web/src/components/workflow/ImportModal.tsx | 18 ++++- web/src/components/workflow/WorkflowMode.tsx | 80 ++++++++++++++++++++ 2 files changed, 94 insertions(+), 4 deletions(-) create mode 100644 web/src/components/workflow/WorkflowMode.tsx diff --git a/web/src/components/workflow/ImportModal.tsx b/web/src/components/workflow/ImportModal.tsx index 340d0d0..6a10fbc 100644 --- a/web/src/components/workflow/ImportModal.tsx +++ b/web/src/components/workflow/ImportModal.tsx @@ -16,7 +16,7 @@ * limitations under the License. */ -import { memo, useMemo, useRef } from 'react'; +import { memo, useMemo, useRef, useState } from 'react'; import { Form as FinalForm } from 'react-final-form'; import { useRouteMatch } from 'react-router-dom'; import { FormApi } from 'final-form'; @@ -42,6 +42,10 @@ import { ToastLimitText } from '..'; import style from './style.less'; +import WorkflowMode, { + WorkflowModeType, +} from 'components/workflow/WorkflowMode'; + interface Props { visible: boolean; workflowInfo?: HandlersWorkflowItem; @@ -54,12 +58,14 @@ function ImportModal({ visible, workflowInfo, onClose, refetch }: Props) { const match = useRouteMatch<{ workspaceId: string }>(); const isEdit = workflowInfo?.latestVersion.status === 'Success'; const isReimport = workflowInfo?.latestVersion.status === 'Failed'; + const [language, setLanguage] = useState('WDL'); const initialValues = useMemo( () => workflowInfo ? { name: workflowInfo.name, + language: workflowInfo.latestVersion.language, url: workflowInfo.latestVersion.metadata.gitURL, tag: workflowInfo.latestVersion.metadata.gitTag, mainWorkflowPath: workflowInfo.latestVersion.mainWorkflowPath, @@ -166,6 +172,13 @@ function ImportModal({ visible, workflowInfo, onClose, refetch }: Props) { return ( + + + - - WDL - void; + disabled?: boolean; +}) { + return ( + + {RADIO_ARRAY.map(item => { + return ( +
{ + if (disabled || item.disabled) return; + onChange(item.value); + }} + style={{ + padding: '12px 16px', + width: 114, + border: + item.value === value + ? '1px solid #94c2ff' + : '1px solid #e4e8ff', + background: item.value === value ? '#e8f4ff' : 'white', + }} + > +
+
+ {item.title} +
+ + +
+ + +
+ ); + })} + + ); +} + +export default memo(WorkflowMode); \ No newline at end of file From 211b8e43addb5b1ed5610ee7abd4cd86055cffd4 Mon Sep 17 00:00:00 2001 From: Bing64 <625255264@qq.com> Date: Mon, 4 Dec 2023 01:15:45 +0800 Subject: [PATCH 09/18] Fixed path truncation issues in submitting workflow files --- .../submission/infrastructure/client/wes/client.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/internal/context/submission/infrastructure/client/wes/client.go b/internal/context/submission/infrastructure/client/wes/client.go index 0fa9569..382302e 100644 --- a/internal/context/submission/infrastructure/client/wes/client.go +++ b/internal/context/submission/infrastructure/client/wes/client.go @@ -111,14 +111,11 @@ func (i *impl) RunWorkflow(ctx context.Context, req *RunWorkflowRequest) (*RunWo return nil, newBadRequestError("workflowAttachment is empty") } prefix := longestCommonPrefix(filesPath) - if len(filesPath) == 1 { - prefix = fmt.Sprintf("%s/", path.Dir(filesPath[0])) - } // fix bug that func longestCommonPrefix() only return string level longestCommonPrefix // However we need path level longestCommonPrefix. // | Main File |Attachment File|String Level Prefix|Path Level Prefix| // |/app/tasks.wdl| /app/test.wdl | /app/t | /app/ | - if !strings.HasSuffix(prefix, "/") { + if !strings.HasSuffix(prefix, "/") && len(prefix) > 0 { prefix = fmt.Sprintf("%s/", path.Dir(prefix)) } for _, filePath := range filesPath { @@ -192,7 +189,11 @@ func longestCommonPrefix(strs []string) string { case 0: return "" case 1: - return strs[0] + if strings.Contains(strs[0], "/") { + return fmt.Sprintf("%s/", path.Dir(strs[0])) + } else { + return "" + } default: if len(strs[0]) == 0 { return "" From a8b6bd6d307774a08945b9b06a192ff36d8028d2 Mon Sep 17 00:00:00 2001 From: Bing64 <625255264@qq.com> Date: Mon, 4 Dec 2023 01:30:50 +0800 Subject: [PATCH 10/18] fix bug that func longestCommonPrefix() only return string level longestCommonPrefix --- .../infrastructure/client/wes/client.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/internal/context/submission/infrastructure/client/wes/client.go b/internal/context/submission/infrastructure/client/wes/client.go index 382302e..ae1f907 100644 --- a/internal/context/submission/infrastructure/client/wes/client.go +++ b/internal/context/submission/infrastructure/client/wes/client.go @@ -111,10 +111,6 @@ func (i *impl) RunWorkflow(ctx context.Context, req *RunWorkflowRequest) (*RunWo return nil, newBadRequestError("workflowAttachment is empty") } prefix := longestCommonPrefix(filesPath) - // fix bug that func longestCommonPrefix() only return string level longestCommonPrefix - // However we need path level longestCommonPrefix. - // | Main File |Attachment File|String Level Prefix|Path Level Prefix| - // |/app/tasks.wdl| /app/test.wdl | /app/t | /app/ | if !strings.HasSuffix(prefix, "/") && len(prefix) > 0 { prefix = fmt.Sprintf("%s/", path.Dir(prefix)) } @@ -208,11 +204,16 @@ func longestCommonPrefix(strs []string) string { prefix := strs[0][0:minStrLen] for { allFound := true - for i := 1; i < strsLen; i++ { - if strings.Index(strs[i], prefix) != 0 { - prefix = prefix[0 : len(prefix)-1] - allFound = false - break + if !strings.HasSuffix(prefix, "/") { + prefix = prefix[0 : len(prefix)-1] + allFound = false + } else { + for i := 1; i < strsLen; i++ { + if strings.Index(strs[i], prefix) != 0 { + prefix = prefix[0 : len(prefix)-1] + allFound = false + break + } } } if allFound || len(prefix) == 0 { From 24734ea103f5b46a62a81b448a322efb934f3990 Mon Sep 17 00:00:00 2001 From: Bing64 <625255264@qq.com> Date: Mon, 4 Dec 2023 01:47:18 +0800 Subject: [PATCH 11/18] Add WorkflowURL param in RunRequest struct to identify the entrance of workflow in multiple attachments --- .../context/submission/domain/run/event_submit_handler.go | 1 + .../context/submission/infrastructure/client/wes/client.go | 5 +++-- .../context/submission/infrastructure/client/wes/model.go | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/context/submission/domain/run/event_submit_handler.go b/internal/context/submission/domain/run/event_submit_handler.go index 6395f53..9e8340f 100644 --- a/internal/context/submission/domain/run/event_submit_handler.go +++ b/internal/context/submission/domain/run/event_submit_handler.go @@ -65,6 +65,7 @@ func (e *EventHandlerSubmitRun) Handle(ctx context.Context, event *submission.Ev BioosRunIDKey: run.ID, }, WorkflowEngineParameters: event.RunConfig.WorkflowEngineParameters, + WorkflowURL: event.RunConfig.MainWorkflowFilePath, }, WorkflowAttachment: event.RunConfig.WorkflowContents, }) diff --git a/internal/context/submission/infrastructure/client/wes/client.go b/internal/context/submission/infrastructure/client/wes/client.go index ae1f907..e4f7043 100644 --- a/internal/context/submission/infrastructure/client/wes/client.go +++ b/internal/context/submission/infrastructure/client/wes/client.go @@ -121,7 +121,7 @@ func (i *impl) RunWorkflow(ctx context.Context, req *RunWorkflowRequest) (*RunWo } newReq = newReq.SetFileReader(workflowAttachment, filePath[len(prefix):], bytes.NewReader(decodeContent)) } - formData, err := runRequest2FormData(&req.RunRequest) + formData, err := runRequest2FormData(&req.RunRequest, prefix) if err != nil { return nil, newBadRequestError(err.Error()) } @@ -224,7 +224,7 @@ func longestCommonPrefix(strs []string) string { } // runRequest2FormData ... -func runRequest2FormData(req *RunRequest) (map[string]string, error) { +func runRequest2FormData(req *RunRequest, prefix string) (map[string]string, error) { formData := make(map[string]string) if req.WorkflowParams != nil && len(req.WorkflowParams) > 0 { workflowParamsInBytes, err := json.Marshal(req.WorkflowParams) @@ -249,5 +249,6 @@ func runRequest2FormData(req *RunRequest) (map[string]string, error) { } formData[workflowType] = req.WorkflowType formData[workflowTypeVersion] = req.WorkflowTypeVersion + formData[workflowURL] = req.WorkflowURL[len(prefix):] return formData, nil } diff --git a/internal/context/submission/infrastructure/client/wes/model.go b/internal/context/submission/infrastructure/client/wes/model.go index 2969a25..e306238 100644 --- a/internal/context/submission/infrastructure/client/wes/model.go +++ b/internal/context/submission/infrastructure/client/wes/model.go @@ -64,6 +64,7 @@ type RunRequest struct { WorkflowTypeVersion string `json:"workflow_type_version"` Tags map[string]interface{} `json:"tags"` WorkflowEngineParameters map[string]interface{} `json:"workflow_engine_parameters"` + WorkflowURL string `json:"workflow_url"` } // RunStatus ... From 83854fa4503aa708db6c07b4b63d75ae27abd0e1 Mon Sep 17 00:00:00 2001 From: Bing64 <625255264@qq.com> Date: Mon, 4 Dec 2023 03:05:28 +0800 Subject: [PATCH 12/18] Change ExposedOptions type to string to adapt to dynamic parameters(bioctl and grpc remain unchanged and may require further attention) --- .../application/command/submission/command.go | 6 +---- .../application/command/submission/create.go | 18 +++++++------- .../application/query/submission/object.go | 6 +---- .../domain/submission/event_create_handler.go | 15 ++++++------ .../submission/domain/submission/factory.go | 2 +- .../submission/domain/submission/object.go | 6 +---- .../persistence/submission/sql/mapper.go | 24 +++++++------------ .../persistence/submission/sql/po.go | 6 +---- .../interface/grpc/submission_mapper.go | 16 ++++--------- .../interface/hertz/handlers/convert.go | 16 ++----------- .../interface/hertz/handlers/submission_vo.go | 8 ++----- web/src/api/index.ts | 8 ++----- web/src/pages/analysis/AnalysisDetail.tsx | 14 ++++++++++- web/src/pages/analysis/AnalysisTaskDetail.tsx | 14 ++++++++++- web/src/pages/workflow/Run.tsx | 15 ++++++++---- 15 files changed, 78 insertions(+), 96 deletions(-) diff --git a/internal/context/submission/application/command/submission/command.go b/internal/context/submission/application/command/submission/command.go index b64b7c2..50c7107 100644 --- a/internal/context/submission/application/command/submission/command.go +++ b/internal/context/submission/application/command/submission/command.go @@ -15,7 +15,7 @@ type CreateSubmissionCommand struct { Description *string `validate:"omitempty,submissionDesc"` Type string `validate:"required,oneof=dataModel filePath"` Entity *Entity - ExposedOptions ExposedOptions + ExposedOptions string InOutMaterial *InOutMaterial } @@ -31,10 +31,6 @@ type InOutMaterial struct { OutputsMaterial string } -type ExposedOptions struct { - ReadFromCache bool -} - type DeleteSubmissionCommand struct { WorkspaceID string `validate:"required"` ID string `validate:"required"` diff --git a/internal/context/submission/application/command/submission/create.go b/internal/context/submission/application/command/submission/create.go index ba7a40b..6d40ed0 100644 --- a/internal/context/submission/application/command/submission/create.go +++ b/internal/context/submission/application/command/submission/create.go @@ -45,16 +45,14 @@ func (c *createSubmissionHandler) Handle(ctx context.Context, cmd *CreateSubmiss } param := submission.CreateSubmissionParam{ - Name: cmd.Name, - Description: cmd.Description, - WorkflowID: cmd.WorkflowID, - WorkspaceID: cmd.WorkspaceID, - Type: cmd.Type, - ExposedOptions: submission.ExposedOptions{ - ReadFromCache: cmd.ExposedOptions.ReadFromCache, - }, - Inputs: make(map[string]interface{}), - Outputs: make(map[string]interface{}), + Name: cmd.Name, + Description: cmd.Description, + WorkflowID: cmd.WorkflowID, + WorkspaceID: cmd.WorkspaceID, + Type: cmd.Type, + ExposedOptions: cmd.ExposedOptions, + Inputs: make(map[string]interface{}), + Outputs: make(map[string]interface{}), } switch param.Type { diff --git a/internal/context/submission/application/query/submission/object.go b/internal/context/submission/application/query/submission/object.go index aee2609..3716b8e 100644 --- a/internal/context/submission/application/query/submission/object.go +++ b/internal/context/submission/application/query/submission/object.go @@ -13,7 +13,7 @@ type SubmissionItem struct { WorkflowVersionID string RunStatus Status Entity *Entity - ExposedOptions ExposedOptions + ExposedOptions string InOutMaterial *InOutMaterial WorkspaceID string } @@ -30,10 +30,6 @@ type InOutMaterial struct { OutputsMaterial string } -type ExposedOptions struct { - ReadFromCache bool -} - type WorkflowVersion struct { ID string VersionID string diff --git a/internal/context/submission/domain/submission/event_create_handler.go b/internal/context/submission/domain/submission/event_create_handler.go index 3e8c589..eb2e6e5 100644 --- a/internal/context/submission/domain/submission/event_create_handler.go +++ b/internal/context/submission/domain/submission/event_create_handler.go @@ -2,11 +2,12 @@ package submission import ( "context" - "reflect" + "encoding/json" "github.com/Bio-OS/bioos/internal/context/workspace/infrastructure/eventbus" workspaceproto "github.com/Bio-OS/bioos/internal/context/workspace/interface/grpc/proto" apperrors "github.com/Bio-OS/bioos/pkg/errors" + applog "github.com/Bio-OS/bioos/pkg/log" "github.com/Bio-OS/bioos/pkg/utils" "github.com/Bio-OS/bioos/pkg/utils/grpc" ) @@ -60,7 +61,7 @@ func (h *CreateHandler) genCreateRunEvent(ctx context.Context, event *CreateEven if err != nil { return nil, apperrors.NewInternalError(err) } - workflowEngineParameters := exposedOptions2Map(&sub.ExposedOptions) + workflowEngineParameters := exposedOptions2Map(sub.ExposedOptions) files, err := h.genWorkflowFiles(ctx, getWorkflowVersionResp.Version, event) if err != nil { return nil, err @@ -99,12 +100,12 @@ func (h *CreateHandler) genWorkflowFiles(ctx context.Context, workflowVersion *w } // exposedOptions2Map ... -func exposedOptions2Map(exposedOptions *ExposedOptions) map[string]interface{} { +func exposedOptions2Map(exposedOptions string) map[string]interface{} { res := make(map[string]interface{}) - t := reflect.TypeOf(exposedOptions).Elem() - v := reflect.ValueOf(exposedOptions).Elem() - for i := 0; i < v.NumField(); i++ { - res[t.Field(i).Tag.Get(WESTag)] = v.Field(i).Interface() + err := json.Unmarshal([]byte(exposedOptions), &res) + if err != nil { + applog.Errorf("invalid exposedOptions, value : ", exposedOptions) + res = make(map[string]interface{}) } return res } diff --git a/internal/context/submission/domain/submission/factory.go b/internal/context/submission/domain/submission/factory.go index 5d37a47..2cec295 100644 --- a/internal/context/submission/domain/submission/factory.go +++ b/internal/context/submission/domain/submission/factory.go @@ -20,7 +20,7 @@ type CreateSubmissionParam struct { Type string Inputs map[string]interface{} Outputs map[string]interface{} - ExposedOptions ExposedOptions + ExposedOptions string } func (p CreateSubmissionParam) validate() error { diff --git a/internal/context/submission/domain/submission/object.go b/internal/context/submission/domain/submission/object.go index 541a75e..67efec1 100644 --- a/internal/context/submission/domain/submission/object.go +++ b/internal/context/submission/domain/submission/object.go @@ -15,12 +15,8 @@ type Submission struct { Type string Inputs map[string]interface{} Outputs map[string]interface{} - ExposedOptions ExposedOptions + ExposedOptions string Status string StartTime time.Time FinishTime *time.Time } - -type ExposedOptions struct { - ReadFromCache bool `wes:"read_from_cache"` -} diff --git a/internal/context/submission/infrastructure/persistence/submission/sql/mapper.go b/internal/context/submission/infrastructure/persistence/submission/sql/mapper.go index cdc9a4c..ba2529b 100644 --- a/internal/context/submission/infrastructure/persistence/submission/sql/mapper.go +++ b/internal/context/submission/infrastructure/persistence/submission/sql/mapper.go @@ -23,9 +23,7 @@ func SubmissionPOToSubmissionDTO(ctx context.Context, submission *Submission) (* WorkflowID: submission.WorkflowID, WorkflowVersionID: submission.WorkflowVersionID, WorkspaceID: submission.WorkspaceID, - ExposedOptions: query.ExposedOptions{ - ReadFromCache: submission.ExposedOptions.ReadFromCache, - }, + ExposedOptions: submission.ExposedOptions, } if submission.FinishTime != nil { item.FinishTime = utils.PointInt64(submission.FinishTime.Unix()) @@ -93,12 +91,10 @@ func SubmissionPOToSubmissionDO(ctx context.Context, sb *Submission) (*submissio Type: sb.Type, Inputs: sb.Inputs, Outputs: sb.Outputs, - ExposedOptions: submission.ExposedOptions{ - ReadFromCache: sb.ExposedOptions.ReadFromCache, - }, - Status: sb.Status, - StartTime: sb.StartTime, - FinishTime: sb.FinishTime, + ExposedOptions: sb.ExposedOptions, + Status: sb.Status, + StartTime: sb.StartTime, + FinishTime: sb.FinishTime, }, nil } @@ -123,11 +119,9 @@ func SubmissionDOToSubmissionPO(ctx context.Context, sb *submission.Submission) Inputs: sb.Inputs, Outputs: sb.Outputs, WorkspaceID: sb.WorkspaceID, - ExposedOptions: ExposedOptions{ - ReadFromCache: sb.ExposedOptions.ReadFromCache, - }, - Status: sb.Status, - StartTime: sb.StartTime, - FinishTime: sb.FinishTime, + ExposedOptions: sb.ExposedOptions, + Status: sb.Status, + StartTime: sb.StartTime, + FinishTime: sb.FinishTime, }, nil } diff --git a/internal/context/submission/infrastructure/persistence/submission/sql/po.go b/internal/context/submission/infrastructure/persistence/submission/sql/po.go index d2158b4..a15c675 100644 --- a/internal/context/submission/infrastructure/persistence/submission/sql/po.go +++ b/internal/context/submission/infrastructure/persistence/submission/sql/po.go @@ -24,17 +24,13 @@ type Submission struct { Type string `gorm:"type:varchar(32);not null"` Inputs map[string]interface{} `gorm:"serializer:json"` Outputs map[string]interface{} `gorm:"serializer:json"` - ExposedOptions ExposedOptions `gorm:"serializer:json"` + ExposedOptions string `gorm:"type:longtext"` Status string `gorm:"type:varchar(32);not null"` StartTime time.Time `gorm:"not null"` FinishTime *time.Time UserID *int64 } -type ExposedOptions struct { - ReadFromCache bool `json:"readFromCache"` -} - func (s *SubmissionModel) TableName() string { return "submission" } diff --git a/internal/context/submission/interface/grpc/submission_mapper.go b/internal/context/submission/interface/grpc/submission_mapper.go index 3385efd..96421e5 100644 --- a/internal/context/submission/interface/grpc/submission_mapper.go +++ b/internal/context/submission/interface/grpc/submission_mapper.go @@ -64,10 +64,8 @@ func queryEntityDTOToVO(entity *query.Entity) *pb.Entity { } } -func queryExposedOptionsDTOToVO(options query.ExposedOptions) *pb.ExposedOptions { - return &pb.ExposedOptions{ - ReadFromCache: options.ReadFromCache, - } +func queryExposedOptionsDTOToVO(options string) *pb.ExposedOptions { + return &pb.ExposedOptions{} } func queryInOutMaterialDTOToVO(material *query.InOutMaterial) *pb.InOutMaterial { @@ -105,13 +103,9 @@ func commandEntityVoToDto(entity *pb.Entity) *command.Entity { } } -func commandExposedOptionsVoToDto(options *pb.ExposedOptions) command.ExposedOptions { - if options == nil { - return command.ExposedOptions{} - } - return command.ExposedOptions{ - ReadFromCache: options.ReadFromCache, - } +func commandExposedOptionsVoToDto(options *pb.ExposedOptions) string { + return "" + } func commandInOutMaterialVoToDto(material *pb.InOutMaterial) *command.InOutMaterial { diff --git a/internal/context/submission/interface/hertz/handlers/convert.go b/internal/context/submission/interface/hertz/handlers/convert.go index 911ba7b..5f72fb9 100644 --- a/internal/context/submission/interface/hertz/handlers/convert.go +++ b/internal/context/submission/interface/hertz/handlers/convert.go @@ -16,7 +16,7 @@ func createSubmissionVoToDto(req CreateSubmissionRequest) *submissioncommand.Cre Description: req.Description, Type: req.Type, Entity: commandEntityVoToDto(req.Entity), - ExposedOptions: commandExposedOptionsVoToDto(req.ExposedOptions), + ExposedOptions: req.ExposedOptions, InOutMaterial: commandInOutMaterialVoToDto(req.InOutMaterial), } } @@ -33,12 +33,6 @@ func commandEntityVoToDto(entity *Entity) *submissioncommand.Entity { } } -func commandExposedOptionsVoToDto(options ExposedOptions) submissioncommand.ExposedOptions { - return submissioncommand.ExposedOptions{ - ReadFromCache: options.ReadFromCache, - } -} - func commandInOutMaterialVoToDto(material *InOutMaterial) *submissioncommand.InOutMaterial { if material == nil { return nil @@ -109,7 +103,7 @@ func submissionItemDtoToVo(item *submissionquery.SubmissionItem) SubmissionItem WorkflowVersion: queryWorkflowVersionDtoToVo(item.WorkflowID, item.WorkflowVersionID), RunStatus: submissionQueryStatusDtoToVo(item.RunStatus), Entity: queryEntityDtoToVo(item.Entity), - ExposedOptions: queryExposedOptionsDtoToVo(item.ExposedOptions), + ExposedOptions: item.ExposedOptions, InOutMaterial: queryInOutMaterialDtoToVo(item.InOutMaterial), } } @@ -145,12 +139,6 @@ func queryEntityDtoToVo(entity *submissionquery.Entity) *Entity { } } -func queryExposedOptionsDtoToVo(options submissionquery.ExposedOptions) ExposedOptions { - return ExposedOptions{ - ReadFromCache: options.ReadFromCache, - } -} - func queryInOutMaterialDtoToVo(material *submissionquery.InOutMaterial) *InOutMaterial { if material == nil { return nil diff --git a/internal/context/submission/interface/hertz/handlers/submission_vo.go b/internal/context/submission/interface/hertz/handlers/submission_vo.go index a1bdea4..54f1812 100644 --- a/internal/context/submission/interface/hertz/handlers/submission_vo.go +++ b/internal/context/submission/interface/hertz/handlers/submission_vo.go @@ -7,7 +7,7 @@ type CreateSubmissionRequest struct { Description *string `json:"description"` Type string `json:"type"` Entity *Entity `json:"entity"` - ExposedOptions ExposedOptions `json:"exposedOptions"` + ExposedOptions string `json:"exposedOptions"` InOutMaterial *InOutMaterial `json:"inOutMaterial"` } @@ -33,10 +33,6 @@ type InOutMaterial struct { OutputsMaterial string `json:"outputsMaterial"` } -type ExposedOptions struct { - ReadFromCache bool `json:"readFromCache"` -} - type CreateSubmissionResponse struct { ID string `json:"id"` } @@ -81,7 +77,7 @@ type SubmissionItem struct { WorkflowVersion WorkflowVersion `json:"workflowVersion"` RunStatus Status `json:"runStatus"` Entity *Entity `json:"entity"` - ExposedOptions ExposedOptions `json:"exposedOptions"` + ExposedOptions string `json:"exposedOptions"` InOutMaterial *InOutMaterial `json:"inOutMaterial"` } diff --git a/web/src/api/index.ts b/web/src/api/index.ts index 1edbb59..13b300e 100644 --- a/web/src/api/index.ts +++ b/web/src/api/index.ts @@ -67,7 +67,7 @@ export interface GithubComBioOSBioosInternalContextWorkspaceInterfaceHertzHandle export interface HandlersCreateSubmissionRequest { description?: string; entity?: HandlersEntity; - exposedOptions?: HandlersExposedOptions; + exposedOptions?: string; inOutMaterial?: HandlersInOutMaterial; name?: string; type?: string; @@ -115,10 +115,6 @@ export interface HandlersEntity { outputsTemplate?: string; } -export interface HandlersExposedOptions { - readFromCache?: boolean; -} - export interface HandlersGetDataModelResponse { dataModel?: HandlersDataModel; headers?: string[]; @@ -233,7 +229,7 @@ export interface HandlersSubmissionItem { description?: string; duration?: number; entity?: HandlersEntity; - exposedOptions?: HandlersExposedOptions; + exposedOptions?: string; finishTime?: number; id?: string; inOutMaterial?: HandlersInOutMaterial; diff --git a/web/src/pages/analysis/AnalysisDetail.tsx b/web/src/pages/analysis/AnalysisDetail.tsx index 0e1321e..95c0e45 100644 --- a/web/src/pages/analysis/AnalysisDetail.tsx +++ b/web/src/pages/analysis/AnalysisDetail.tsx @@ -129,6 +129,18 @@ export default function AnalysisDetail() { (detailData?.status === 'Pending' ? 'Running' : detailData?.status), ); + const parsedExposedOptions = useMemo(() => { + if (detailData?.exposedOptions) { + try { + return JSON.parse(detailData.exposedOptions); + } catch (e) { + console.error("Parsing exposedOptions failed", e); + return {}; + } + } + return {}; + }, [detailData]); + //获取 icon 组件 const getIcon = () => { switch (status?.icon) { @@ -199,7 +211,7 @@ export default function AnalysisDetail() { id={detailData?.id} workspaceId={workspaceId} workflowId={detailData?.workflowVersion?.id} - flagReadFromCache={detailData?.exposedOptions?.readFromCache} + flagReadFromCache={parsedExposedOptions?.readFromCache} /> diff --git a/web/src/pages/analysis/AnalysisTaskDetail.tsx b/web/src/pages/analysis/AnalysisTaskDetail.tsx index 0f64298..4ab8d46 100644 --- a/web/src/pages/analysis/AnalysisTaskDetail.tsx +++ b/web/src/pages/analysis/AnalysisTaskDetail.tsx @@ -113,6 +113,18 @@ export default function AnalysisTaskDetail() { const status = RUN_STATUS_TAG.find(item => item.value === runData.status); + const parsedExposedOptions = useMemo(() => { + if (runSubmissionData?.exposedOptions) { + try { + return JSON.parse(runSubmissionData.exposedOptions); + } catch (e) { + console.error("Parsing exposedOptions failed", e); + return {}; + } + } + return {}; + }, [runSubmissionData]); + useEffect(() => { startAnalysisDetailPolling(); return () => stopAnalysisDetailPolling(); @@ -146,7 +158,7 @@ export default function AnalysisTaskDetail() { inputs={runData.inputs} outputs={runData.outputs} logs={logs} - callCache={runSubmissionData?.exposedOptions?.readFromCache} + callCache={parsedExposedOptions?.readFromCache} isFinished={isFinished} /> diff --git a/web/src/pages/workflow/Run.tsx b/web/src/pages/workflow/Run.tsx index 5ccd6c5..6701d16 100644 --- a/web/src/pages/workflow/Run.tsx +++ b/web/src/pages/workflow/Run.tsx @@ -308,11 +308,12 @@ export default function WorkflowRun() { workspaceID: workspaceId, workflowID: workflowId, type: isPath ? 'filePath' : 'dataModel', - exposedOptions: { - readFromCache: callCaching, - }, }; + body.exposedOptions = JSON.stringify({ + readFromCache: callCaching, + }) + if (isPath) { body.inOutMaterial = { inputsMaterial: JSON.stringify(inputJsonObj), @@ -515,7 +516,13 @@ export default function WorkflowRun() { useEffect(() => { if (!dataModelList) return; - setCallCaching(submission?.exposedOptions?.readFromCache ?? true); + try { + const exposedOptions = submission?.exposedOptions ? JSON.parse(submission.exposedOptions) : {}; + setCallCaching(exposedOptions.readFromCache ?? true); + } catch (e) { + setCallCaching(true); // 解析失败时，使用默认值true + } + if (!submissionId || submission?.type === 'filePath') { setModelId(dataModelList?.[0]?.id); setMode(submission?.type || 'dataModel'); From 2780a3cc841bbf10dab3e3ec6ee5f5fca9ae4e0a Mon Sep 17 00:00:00 2001 From: Bing64 <625255264@qq.com> Date: Mon, 4 Dec 2023 11:18:48 +0800 Subject: [PATCH 13/18] Modify Dockerfile: install cwltool, change user --- build/apiserver/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build/apiserver/Dockerfile b/build/apiserver/Dockerfile index f7d87b9..1df3268 100644 --- a/build/apiserver/Dockerfile +++ b/build/apiserver/Dockerfile @@ -13,11 +13,13 @@ RUN apt update \ && apt install -y --no-install-recommends ca-certificates openjdk-11-jre-headless \ && apt clean \ && rm -rf /var/lib/apt/lists/* +RUN apt-get update \ + && apt-get install -y cwltool ARG TARGETOS ARG TARGETARCH -USER nobody +USER root WORKDIR /app -COPY --from=builder --chown=nobody:nogroup /go/src/github.com/Bio-OS/bioos/conf conf +COPY --from=builder --chown=root:root /go/src/github.com/Bio-OS/bioos/conf conf COPY --from=builder /go/src/github.com/Bio-OS/bioos/_output/platforms/${TARGETOS}/${TARGETARCH}/apiserver . COPY --from=builder /go/src/github.com/Bio-OS/bioos/womtool.jar . ENTRYPOINT ["/app/apiserver"] From 828c25dc7574b902556d0513458defe9178b31c7 Mon Sep 17 00:00:00 2001 From: Bing64 <625255264@qq.com> Date: Mon, 4 Dec 2023 11:25:32 +0800 Subject: [PATCH 14/18] Fix variable mistake --- internal/context/submission/domain/run/event_submit_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/context/submission/domain/run/event_submit_handler.go b/internal/context/submission/domain/run/event_submit_handler.go index 9e8340f..84cf54d 100644 --- a/internal/context/submission/domain/run/event_submit_handler.go +++ b/internal/context/submission/domain/run/event_submit_handler.go @@ -86,7 +86,7 @@ func (e *EventHandlerSubmitRun) markRunFailed(ctx context.Context, run *Run, mes tempRun := run.Copy() tempRun.Message = utils.PointString(message) tempRun.FinishTime = utils.PointTime(time.Now()) - if err := e.runRepo.Save(ctx, run); err != nil { + if err := e.runRepo.Save(ctx, tempRun); err != nil { return apperrors.NewInternalError(err) } return nil From 449a9dcbdaff414ca25d95acd4781eaa139e4c79 Mon Sep 17 00:00:00 2001 From: Bing64 <625255264@qq.com> Date: Mon, 4 Dec 2023 14:12:34 +0800 Subject: [PATCH 15/18] Add log in GetRunLog --- .../submission/infrastructure/client/wes/client.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/context/submission/infrastructure/client/wes/client.go b/internal/context/submission/infrastructure/client/wes/client.go index e4f7043..13fb33d 100644 --- a/internal/context/submission/infrastructure/client/wes/client.go +++ b/internal/context/submission/infrastructure/client/wes/client.go @@ -153,6 +153,7 @@ func (i *impl) GetRunLog(ctx context.Context, req *GetRunLogRequest) (*GetRunLog if res.StatusCode >= 400 { return nil, res.ErrorResp } + UpdateLogsWithStdout(&res.GetRunLogResponse) return &res.GetRunLogResponse, nil } else if resp.IsError() { return nil, res.ErrorResp @@ -252,3 +253,11 @@ func runRequest2FormData(req *RunRequest, prefix string) (map[string]string, err formData[workflowURL] = req.WorkflowURL[len(prefix):] return formData, nil } + +// 标准WES API中没有log字段，在此处处理 +func UpdateLogsWithStdout(response *GetRunLogResponse) { + response.RunLog.Log = response.RunLog.Stdout + for index := range response.TaskLogs { + response.TaskLogs[index].Log = response.TaskLogs[index].Stdout + } +} From e56871e0699185a051428fa706a1794c87c412d2 Mon Sep 17 00:00:00 2001 From: Bing64 <625255264@qq.com> Date: Mon, 4 Dec 2023 22:35:13 +0800 Subject: [PATCH 16/18] Add workflow language display --- web/src/pages/workflow/Run.tsx | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/web/src/pages/workflow/Run.tsx b/web/src/pages/workflow/Run.tsx index 6701d16..4a6bc48 100644 --- a/web/src/pages/workflow/Run.tsx +++ b/web/src/pages/workflow/Run.tsx @@ -312,7 +312,7 @@ export default function WorkflowRun() { body.exposedOptions = JSON.stringify({ readFromCache: callCaching, - }) + }); if (isPath) { body.inOutMaterial = { @@ -642,6 +642,18 @@ export default function WorkflowRun() { > {workflow?.description} + 流程语言： + + {workflow?.latestVersion?.language} + 来源： {workflow?.latestVersion?.metadata?.gitURL} From c9a5e276fd4adfc3c7ade4115cc38cded074d5ca Mon Sep 17 00:00:00 2001 From: Bing64 <625255264@qq.com> Date: Mon, 4 Dec 2023 23:02:40 +0800 Subject: [PATCH 17/18] Add " to Directory type when import json file --- web/src/components/workflow/run/UploadJSON.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/web/src/components/workflow/run/UploadJSON.tsx b/web/src/components/workflow/run/UploadJSON.tsx index 7c47d97..8018158 100644 --- a/web/src/components/workflow/run/UploadJSON.tsx +++ b/web/src/components/workflow/run/UploadJSON.tsx @@ -81,10 +81,12 @@ export default function UploadJSON({ let jsonVal = jsonObj[item.name]; // String和File类型需要多加一层双引号 + // Directory类型(CWL流程中会出现)也需要加引号 if ( jsonVal && (item?.type.startsWith('String') || - item?.type.startsWith('File')) && + item?.type.startsWith('File') || + item?.type.startsWith('Directory')) && !(jsonVal?.startsWith('"') && jsonVal?.endsWith('"')) && !isContextValue(jsonVal) ) { From 67b279f472ae5615247e9cf01578b8fba8f60e63 Mon Sep 17 00:00:00 2001 From: Bing64 <625255264@qq.com> Date: Tue, 5 Dec 2023 01:20:26 +0800 Subject: [PATCH 18/18] Add different exposed options for different languages --- web/src/pages/workflow/Run.tsx | 148 +++++++++++++++++++++++++++------ 1 file changed, 122 insertions(+), 26 deletions(-) diff --git a/web/src/pages/workflow/Run.tsx b/web/src/pages/workflow/Run.tsx index 4a6bc48..d7032c4 100644 --- a/web/src/pages/workflow/Run.tsx +++ b/web/src/pages/workflow/Run.tsx @@ -310,9 +310,7 @@ export default function WorkflowRun() { type: isPath ? 'filePath' : 'dataModel', }; - body.exposedOptions = JSON.stringify({ - readFromCache: callCaching, - }); + body.exposedOptions = JSON.stringify(exposedOptions); if (isPath) { body.inOutMaterial = { @@ -505,6 +503,59 @@ export default function WorkflowRun() { } } + function useLanguageOptions(language, submission) { + // 定义默认状态 + const [optionStates, setOptionStates] = useState({ + callCaching: true, + continueWhenFailed: true, + useConda: true, + // ...在此添加状态 + }); + + // 根据状态对象生成 exposedOptions + const exposedOptions = useMemo(() => { + switch (language) { + case 'WDL': + const WDLOptions = { + readFromCache: optionStates.callCaching, + }; + return WDLOptions; + case 'CWL': + const CWLOptions = { + workflowFailureMode: optionStates.continueWhenFailed ? 'ContinueWhilePossible' : 'NoNewCalls', + }; + return CWLOptions; + case 'SMK': + const SMKOptions = { + useConda: optionStates.useConda, + }; + return SMKOptions; + default: + return {}; + } + }, [optionStates, language]); + + // 通用的更新方法 + const updateOptionState = (name, newValue) => { + setOptionStates(prevState => ({ + ...prevState, + [name]: newValue, + })); + }; + + // 处理 submission 变化，更新状态对象 + useEffect(() => { + if (submission) { + const submissionOptions = submission?.exposedOptions ? JSON.parse(submission.exposedOptions) : {}; + Object.keys(submissionOptions).forEach(key => { + updateOptionState(key, submissionOptions[key]); + }); + } + }, [submission]); + + return { optionStates, updateOptionState, exposedOptions }; + } + // 获取初始数据 useEffect(() => { getWorkflow(); @@ -514,14 +565,75 @@ export default function WorkflowRun() { } }, []); + const language = workflow?.latestVersion?.language; + const { optionStates, updateOptionState, exposedOptions } = useLanguageOptions(language, submission); + + const getLanguageOptionsUI = (language, optionStates, updateOptionState) => { + switch (language) { + case 'WDL': + return ( +
+
+ CallCaching + + + +
+
+ updateOptionState('callCaching', checked)} + /> +
+
+ ); + case 'CWL': + return ( +
+
+ FailMode + + + +
+
+ updateOptionState('continueWhenFailed', checked)} + /> +
+
+ ); + case 'SMK': + return ( +
+
+ UseConda + + + +
+
+ updateOptionState('useConda', checked)} + /> +
+
+ ); + default: + return
No options available for this language.
; + } + }; + useEffect(() => { if (!dataModelList) return; - try { - const exposedOptions = submission?.exposedOptions ? JSON.parse(submission.exposedOptions) : {}; - setCallCaching(exposedOptions.readFromCache ?? true); - } catch (e) { - setCallCaching(true); // 解析失败时，使用默认值true - } if (!submissionId || submission?.type === 'filePath') { setModelId(dataModelList?.[0]?.id); @@ -725,23 +837,7 @@ export default function WorkflowRun() { )} - -
-
- CallCaching - - - -
-
- -
-
+ {getLanguageOptionsUI(language, optionStates, updateOptionState)}