diff --git a/.gitignore b/.gitignore index 559d55941..e0203585e 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ util/version.go /coverage.out /public/package-lock.json !.gitkeep + +.dredd/compiled_hooks \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 900578b5d..1e98fe7ef 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,10 +96,8 @@ This means that you should never run these tests against your productive databas The best practice to run these tests is to use docker and the task commands. ```bash -task dc:build #First run only to build the images -context=dev task dc:up -task test:api -# alternatively if you want to run dredd in a container use the following command. -# You will need to use the host network so that it can reach the docker container on a 0.0.0.0 address -# docker run -it --rm -v $(pwd):/home/developer/src --network host tomwhiston/dredd --config .dredd/dredd.yml +context=dev task dc:build #build fresh semaphore images +context=dev task dc:up #up semaphore and mysql +task dc:build:dredd #build fresh dredd image +task dc:up:dredd #run dredd over docker-compose stack ``` \ No newline at end of file diff --git a/Gopkg.lock b/Gopkg.lock index bca322d13..ed8ceec94 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -27,9 +27,9 @@ [[projects]] branch = "master" - name = "github.com/castawaylabs/mulekick" + name = "github.com/strangeman/mulekick" packages = ["."] - revision = "7029fb389811e0f873c56cfbbda64d66af48b095" + revision = "3d90b180d25370ed05f3173b93757c53d75883b2" [[projects]] branch = "master" @@ -173,6 +173,12 @@ revision = "a6b93000bd219143c56c16e6cb1c4b91da3f224b" version = "v1.0" +[[projects]] + branch = "master" + name = "github.com/tjarratt/babble" + packages = ["."] + revision = "eecdf8c2339de49c56f0e3ac459077db829be7ee" + [[projects]] branch = "master" name = "github.com/mitchellh/mapstructure" diff --git a/Gopkg.toml b/Gopkg.toml index 0e4bbfff1..f5fe4aec5 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -4,7 +4,7 @@ [[constraint]] branch = "master" - name = "github.com/castawaylabs/mulekick" + name = "github.com/strangeman/mulekick" [[constraint]] name = "github.com/go-sql-driver/mysql" @@ -58,6 +58,10 @@ name = "gopkg.in/ldap.v2" version = "2.5.1" +[[constraint]] + name = "github.com/tjarratt/babble" + branch = "master" + [prune] go-tests = true unused-packages = true diff --git a/Taskfile.yml b/Taskfile.yml index aac43dbfa..507a0aaad 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -100,7 +100,7 @@ tasks: sh: git rev-parse --abbrev-ref HEAD DIRTY: # We must exclude the package-lock file as npm install can change it! - sh: git diff --exit-code --stat -- . ':(exclude)web/package-lock.json' ':(exclude)web/package.json' + sh: git diff --exit-code --stat -- . ':(exclude)web/package-lock.json' ':(exclude)web/package.json' ':(exclude)deployment/*' ':(exclude)Taskfile*' SHA: sh: git log --pretty=format:'%h' -n 1 TIMESTAMP: @@ -241,6 +241,27 @@ tasks: args: stop context: "{{ .context }}" + dc:build:dredd: + desc: build a dredd container to the local testing with docker-compose stack + cmds: + - task: docker + vars: + context: dev + compose: true + prefix: -dredd + args: build + + dc:up:dredd: + desc: build a dredd container to the local testing with docker-compose stack + cmds: + - task: docker + vars: + context: dev + compose: true + prefix: -dredd + args: up + + docker:build: desc: Build an image for Semaphore, requires context vars: @@ -302,4 +323,4 @@ tasks: vars: docker_root: deployment/docker/ cmds: - - docker{{ if .compose }}-compose{{ end }} {{ if .action }}{{ .action }}{{ end }} -f {{ .docker_root }}{{ .context }}/{{ if .compose }}docker-compose.yml{{ else }}Dockerfile{{ end }} {{if .args }}{{ .args }}{{ end }} + - docker{{ if .compose }}-compose{{ end }} {{ if .action }}{{ .action }}{{ end }} -f {{ .docker_root }}{{ .context }}/{{ if .compose }}docker-compose{{ if .prefix }}{{ .prefix }}{{ end }}.yml{{ else }}Dockerfile{{ if .prefix }}{{ .prefix }}{{ end }}{{ end }} {{if .args }}{{ .args }}{{ end }} diff --git a/api-docs.yml b/api-docs.yml index 43d42793b..e6c8a7dd5 100644 --- a/api-docs.yml +++ b/api-docs.yml @@ -287,6 +287,8 @@ definitions: environment_id: type: integer minimum: 1 + slug: + type: string alias: type: string playbook: @@ -301,6 +303,8 @@ definitions: id: type: integer minimum: 1 + slug: + type: string ssh_key_id: type: integer minimum: 1 @@ -424,6 +428,13 @@ parameters: type: integer required: true x-example: 8 + slug: + name: slug + description: Slug + in: path + type: string + required: true + x-example: some-task paths: /ping: @@ -451,7 +462,7 @@ paths: responses: 200: description: OK - 403: + 401: description: not authenticated /info: @@ -1213,6 +1224,34 @@ paths: description: Task queued schema: $ref: "#/definitions/Task" + /project/{project_id}/slug/{slug}: + parameters: + - $ref: "#/parameters/project_id" + - $ref: "#/parameters/slug" + post: + tags: + - project + summary: Starts a job via task slug + parameters: + - name: task + in: body + required: true + schema: + type: object + properties: + debug: + type: boolean + dry_run: + type: boolean + playbook: + type: string + environment: + type: string + responses: + 201: + description: Task queued + schema: + $ref: "#/definitions/Task" /project/{project_id}/tasks/last: parameters: - $ref: "#/parameters/project_id" diff --git a/api/auth.go b/api/auth.go index 3f8bc24eb..ee42aabde 100644 --- a/api/auth.go +++ b/api/auth.go @@ -20,7 +20,7 @@ func authentication(w http.ResponseWriter, r *http.Request) { var token db.APIToken if err := db.Mysql.SelectOne(&token, "select * from user__token where id=? and expired=0", strings.Replace(authHeader, "bearer ", "", 1)); err != nil { if err == sql.ErrNoRows { - w.WriteHeader(http.StatusForbidden) + w.WriteHeader(http.StatusUnauthorized) return } @@ -32,20 +32,20 @@ func authentication(w http.ResponseWriter, r *http.Request) { // fetch session from cookie cookie, err := r.Cookie("semaphore") if err != nil { - w.WriteHeader(http.StatusForbidden) + w.WriteHeader(http.StatusUnauthorized) return } value := make(map[string]interface{}) if err = util.Cookie.Decode("semaphore", cookie.Value, &value); err != nil { - w.WriteHeader(http.StatusForbidden) + w.WriteHeader(http.StatusUnauthorized) return } user, ok := value["user"] sessionVal, okSession := value["session"] if !ok || !okSession { - w.WriteHeader(http.StatusForbidden) + w.WriteHeader(http.StatusUnauthorized) return } @@ -55,7 +55,7 @@ func authentication(w http.ResponseWriter, r *http.Request) { // fetch session var session db.Session if err := db.Mysql.SelectOne(&session, "select * from session where id=? and user_id=? and expired=0", sessionID, userID); err != nil { - w.WriteHeader(http.StatusForbidden) + w.WriteHeader(http.StatusUnauthorized) return } @@ -66,7 +66,7 @@ func authentication(w http.ResponseWriter, r *http.Request) { panic(err) } - w.WriteHeader(http.StatusForbidden) + w.WriteHeader(http.StatusUnauthorized) return } @@ -78,7 +78,7 @@ func authentication(w http.ResponseWriter, r *http.Request) { user, err := db.FetchUser(userID) if err != nil { fmt.Println("Can't find user", err) - w.WriteHeader(http.StatusForbidden) + w.WriteHeader(http.StatusUnauthorized) return } diff --git a/api/events.go b/api/events.go index bc7d057b3..ef22f79a9 100644 --- a/api/events.go +++ b/api/events.go @@ -4,10 +4,10 @@ import ( "net/http" "github.com/ansible-semaphore/semaphore/db" - "github.com/castawaylabs/mulekick" + "github.com/ansible-semaphore/semaphore/util" "github.com/gorilla/context" "github.com/masterminds/squirrel" - "github.com/ansible-semaphore/semaphore/util" + "github.com/strangeman/mulekick" ) //nolint: gocyclo diff --git a/api/login.go b/api/login.go index 8e47510f8..9b007ae24 100644 --- a/api/login.go +++ b/api/login.go @@ -12,7 +12,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" + "github.com/strangeman/mulekick" sq "github.com/masterminds/squirrel" "golang.org/x/crypto/bcrypt" "gopkg.in/ldap.v2" diff --git a/api/projects/environment.go b/api/projects/environment.go index 9e4088eba..6c1947f37 100644 --- a/api/projects/environment.go +++ b/api/projects/environment.go @@ -7,9 +7,9 @@ import ( "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" "github.com/gorilla/context" "github.com/masterminds/squirrel" + "github.com/strangeman/mulekick" ) // EnvironmentMiddleware ensures an environment exists and loads it to the context diff --git a/api/projects/inventory.go b/api/projects/inventory.go index e65ffe7d0..bfbf42454 100644 --- a/api/projects/inventory.go +++ b/api/projects/inventory.go @@ -6,7 +6,7 @@ import ( "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" + "github.com/strangeman/mulekick" "github.com/gorilla/context" "github.com/masterminds/squirrel" "path/filepath" diff --git a/api/projects/keys.go b/api/projects/keys.go index 40eaeb97f..a32116b8c 100644 --- a/api/projects/keys.go +++ b/api/projects/keys.go @@ -6,7 +6,7 @@ import ( "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" + "github.com/strangeman/mulekick" "github.com/gorilla/context" "github.com/masterminds/squirrel" ) diff --git a/api/projects/project.go b/api/projects/project.go index 4ffaddf13..37bc15a0b 100644 --- a/api/projects/project.go +++ b/api/projects/project.go @@ -6,7 +6,7 @@ import ( "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" + "github.com/strangeman/mulekick" "github.com/gorilla/context" "github.com/masterminds/squirrel" ) diff --git a/api/projects/projects.go b/api/projects/projects.go index f0d868ac2..daabf1396 100644 --- a/api/projects/projects.go +++ b/api/projects/projects.go @@ -4,7 +4,7 @@ import ( "net/http" "github.com/ansible-semaphore/semaphore/db" - "github.com/castawaylabs/mulekick" + "github.com/strangeman/mulekick" "github.com/gorilla/context" "github.com/masterminds/squirrel" "time" diff --git a/api/projects/repository.go b/api/projects/repository.go index 3e8b1f3f4..81bce899e 100644 --- a/api/projects/repository.go +++ b/api/projects/repository.go @@ -8,7 +8,7 @@ import ( "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" + "github.com/strangeman/mulekick" "github.com/gorilla/context" "github.com/masterminds/squirrel" ) diff --git a/api/projects/templates.go b/api/projects/templates.go index ec82c7f04..33fa718ee 100644 --- a/api/projects/templates.go +++ b/api/projects/templates.go @@ -4,12 +4,14 @@ import ( "database/sql" "net/http" "strconv" + "strings" "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" "github.com/gorilla/context" "github.com/masterminds/squirrel" + "github.com/strangeman/mulekick" + "github.com/tjarratt/babble" ) // TemplatesMiddleware ensures a template exists and loads it to the context @@ -46,21 +48,22 @@ func GetTemplates(w http.ResponseWriter, r *http.Request) { } q := squirrel.Select("pt.id", - "pt.ssh_key_id", - "pt.project_id", - "pt.inventory_id", - "pt.repository_id", - "pt.environment_id", - "pt.alias", - "pt.playbook", - "pt.arguments", - "pt.override_args"). - From("project__template pt") + "pt.slug", + "pt.ssh_key_id", + "pt.project_id", + "pt.inventory_id", + "pt.repository_id", + "pt.environment_id", + "pt.alias", + "pt.playbook", + "pt.arguments", + "pt.override_args"). + From("project__template pt") switch sort { case "alias", "playbook": q = q.Where("pt.project_id=?", project.ID). - OrderBy("pt."+ sort + " " + order) + OrderBy("pt." + sort + " " + order) case "ssh_key": q = q.LeftJoin("access_key ak ON (pt.ssh_key_id = ak.id)"). Where("pt.project_id=?", project.ID). @@ -79,7 +82,7 @@ func GetTemplates(w http.ResponseWriter, r *http.Request) { OrderBy("pr.name " + order) default: q = q.Where("pt.project_id=?", project.ID). - OrderBy("pt.alias " + order) + OrderBy("pt.alias " + order) } query, args, err := q.ToSql() @@ -101,8 +104,16 @@ func AddTemplate(w http.ResponseWriter, r *http.Request) { return } - res, err := db.Mysql.Exec("insert into project__template set ssh_key_id=?, project_id=?, inventory_id=?, repository_id=?, environment_id=?, alias=?, playbook=?, arguments=?, override_args=?", template.SSHKeyID, project.ID, template.InventoryID, template.RepositoryID, template.EnvironmentID, template.Alias, template.Playbook, template.Arguments, template.OverrideArguments) + if template.Slug == "" { + template.Slug = babble.NewBabbler().Babble() + } + + res, err := db.Mysql.Exec("insert into project__template set slug=?, ssh_key_id=?, project_id=?, inventory_id=?, repository_id=?, environment_id=?, alias=?, playbook=?, arguments=?, override_args=?", template.Slug, template.SSHKeyID, project.ID, template.InventoryID, template.RepositoryID, template.EnvironmentID, template.Alias, template.Playbook, template.Arguments, template.OverrideArguments) if err != nil { + if strings.Contains(err.Error(), "Error 1062") { + mulekick.WriteJSON(w, http.StatusBadRequest, "{'error': 'slug must be unique'}") + return + } panic(err) } @@ -140,7 +151,11 @@ func UpdateTemplate(w http.ResponseWriter, r *http.Request) { template.Arguments = nil } - if _, err := db.Mysql.Exec("update project__template set ssh_key_id=?, inventory_id=?, repository_id=?, environment_id=?, alias=?, playbook=?, arguments=?, override_args=? where id=?", template.SSHKeyID, template.InventoryID, template.RepositoryID, template.EnvironmentID, template.Alias, template.Playbook, template.Arguments, template.OverrideArguments, oldTemplate.ID); err != nil { + if template.Slug == "" { + template.Slug = babble.NewBabbler().Babble() + } + + if _, err := db.Mysql.Exec("update project__template set slug=?, ssh_key_id=?, inventory_id=?, repository_id=?, environment_id=?, alias=?, playbook=?, arguments=?, override_args=? where id=?", template.Slug, template.SSHKeyID, template.InventoryID, template.RepositoryID, template.EnvironmentID, template.Alias, template.Playbook, template.Arguments, template.OverrideArguments, oldTemplate.ID); err != nil { panic(err) } diff --git a/api/projects/users.go b/api/projects/users.go index 7c5275f84..823149e7b 100644 --- a/api/projects/users.go +++ b/api/projects/users.go @@ -7,7 +7,7 @@ import ( "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" + "github.com/strangeman/mulekick" "github.com/gorilla/context" "github.com/masterminds/squirrel" ) diff --git a/api/router.go b/api/router.go index b89984373..71645f41f 100644 --- a/api/router.go +++ b/api/router.go @@ -8,10 +8,10 @@ import ( "github.com/ansible-semaphore/semaphore/api/sockets" "github.com/ansible-semaphore/semaphore/api/tasks" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" "github.com/gobuffalo/packr" "github.com/gorilla/mux" "github.com/russross/blackfriday" + "github.com/strangeman/mulekick" ) var publicAssets = packr.NewBox("../web/public") @@ -25,6 +25,7 @@ func JSONMiddleware(w http.ResponseWriter, r *http.Request) { func PlainTextMiddleware(w http.ResponseWriter, r *http.Request) { w.Header().Set("content-type", "text/plain; charset=utf-8") } + // Route declares all routes func Route() mulekick.Router { r := mulekick.New(mux.NewRouter(), mulekick.CorsMiddleware, JSONMiddleware) @@ -110,6 +111,8 @@ func Route() mulekick.Router { api.Put("/templates/{template_id}", projects.TemplatesMiddleware, projects.UpdateTemplate) api.Delete("/templates/{template_id}", projects.TemplatesMiddleware, projects.RemoveTemplate) + api.Post("/slug/{slug}", tasks.AddTask) + api.Get("/tasks", tasks.GetAllTasks) api.Get("/tasks/last", tasks.GetLastTasks) api.Post("/tasks", tasks.AddTask) diff --git a/api/tasks/http.go b/api/tasks/http.go index 8d3175c19..03b69e58f 100644 --- a/api/tasks/http.go +++ b/api/tasks/http.go @@ -5,19 +5,24 @@ import ( "strconv" "time" + "database/sql" + log "github.com/Sirupsen/logrus" "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" "github.com/gorilla/context" + "github.com/gorilla/mux" "github.com/masterminds/squirrel" + "github.com/strangeman/mulekick" ) -// AddTask inserts a task into the database and returns a header or panics +// AddTask inserts a task into the database and returns a header or returns error func AddTask(w http.ResponseWriter, r *http.Request) { project := context.Get(r, "project").(db.Project) user := context.Get(r, "user").(*db.User) + slug := mux.Vars(r)["slug"] + var taskObj db.Task if err := mulekick.Bind(w, r, &taskObj); err != nil { return @@ -27,8 +32,23 @@ func AddTask(w http.ResponseWriter, r *http.Request) { taskObj.Status = "waiting" taskObj.UserID = &user.ID + if slug != "" { + var template db.Template + if err := db.Mysql.SelectOne(&template, "select * from project__template where project_id=? and slug=?", project.ID, slug); err != nil { + if err == sql.ErrNoRows { + mulekick.WriteJSON(w, http.StatusNotFound, nil) + return + } + } else { + println(template.ID) + taskObj.TemplateID = template.ID + } + } + if err := db.Mysql.Insert(&taskObj); err != nil { - panic(err) + util.LogErrorWithFields(err, log.Fields{"error": "Bad request. Cannot create new task"}) + w.WriteHeader(http.StatusBadRequest) + return } pool.register <- &task{ @@ -44,13 +64,13 @@ func AddTask(w http.ResponseWriter, r *http.Request) { ObjectID: &taskObj.ID, Description: &desc, }.Insert()); err != nil { - panic(err) + util.LogErrorWithFields(err, log.Fields{"error": "Cannot write new event to database"}) } mulekick.WriteJSON(w, http.StatusCreated, taskObj) } -// GetTasksList returns a list of tasks for the current project in desc order to limit or panics +// GetTasksList returns a list of tasks for the current project in desc order to limit or error func GetTasksList(w http.ResponseWriter, r *http.Request, limit uint64) { project := context.Get(r, "project").(db.Project) @@ -75,7 +95,9 @@ func GetTasksList(w http.ResponseWriter, r *http.Request, limit uint64) { UserName *string `db:"user_name" json:"user_name"` } if _, err := db.Mysql.Select(&tasks, query, args...); err != nil { - panic(err) + util.LogErrorWithFields(err, log.Fields{"error": "Bad request. Cannot get tasks list from database"}) + w.WriteHeader(http.StatusBadRequest) + return } mulekick.WriteJSON(w, http.StatusOK, tasks) @@ -112,13 +134,15 @@ func GetTaskMiddleware(w http.ResponseWriter, r *http.Request) { context.Set(r, taskTypeID, task) } -// GetTaskOutput returns the logged task output by id and writes it as json +// GetTaskOutput returns the logged task output by id and writes it as json or returns error func GetTaskOutput(w http.ResponseWriter, r *http.Request) { task := context.Get(r, taskTypeID).(db.Task) var output []db.TaskOutput if _, err := db.Mysql.Select(&output, "select task_id, task, time, output from task__output where task_id=? order by time asc", task.ID); err != nil { - panic(err) + util.LogErrorWithFields(err, log.Fields{"error": "Bad request. Cannot get task output from database"}) + w.WriteHeader(http.StatusBadRequest) + return } mulekick.WriteJSON(w, http.StatusOK, output) @@ -143,7 +167,9 @@ func RemoveTask(w http.ResponseWriter, r *http.Request) { for _, statement := range statements { _, err := db.Mysql.Exec(statement, task.ID) if err != nil { - panic(err) + util.LogErrorWithFields(err, log.Fields{"error": "Bad request. Cannot delete task from database"}) + w.WriteHeader(http.StatusBadRequest) + return } } diff --git a/api/tasks/pool.go b/api/tasks/pool.go index 61e9bf80a..ebe426646 100644 --- a/api/tasks/pool.go +++ b/api/tasks/pool.go @@ -1,9 +1,10 @@ package tasks import ( - "fmt" + "strconv" "time" + log "github.com/Sirupsen/logrus" "github.com/ansible-semaphore/semaphore/util" ) @@ -74,23 +75,38 @@ func (p *taskPool) run() { for { select { case task := <-p.register: - fmt.Println(task) - go task.prepareRun() p.queue = append(p.queue, task) + log.Debug(task) + msg := "Task " + strconv.Itoa(task.task.ID) + " added to queue" + task.log(msg) + log.Info(msg) case <-ticker.C: if len(p.queue) == 0 { continue - } else if t := p.queue[0]; t.task.Status != taskFailStatus && (!t.prepared || p.blocks(t)) { + } + + //get task from top of queue + t := p.queue[0] + if t.task.Status == taskFailStatus { + //delete failed task from queue + p.queue = p.queue[1:] + log.Info("Task " + strconv.Itoa(t.task.ID) + " removed from queue") + continue + } + if p.blocks(t) { + //move blocked task to end of queue p.queue = append(p.queue[1:], t) continue } - - if t := pool.queue[0]; t.task.Status != taskFailStatus { - fmt.Println("Running a task.") - resourceLocker <- &resourceLock{lock: true, holder: t} - go t.run() + log.Info("Set resourse locker with task " + strconv.Itoa(t.task.ID)) + resourceLocker <- &resourceLock{lock: true, holder: t} + if !t.prepared { + go t.prepareRun() + continue } - pool.queue = pool.queue[1:] + go t.run() + p.queue = p.queue[1:] + log.Info("Task " + strconv.Itoa(t.task.ID) + " removed from queue") } } } diff --git a/api/tasks/runner.go b/api/tasks/runner.go index 339759064..3c2f3e430 100644 --- a/api/tasks/runner.go +++ b/api/tasks/runner.go @@ -14,16 +14,16 @@ import ( "strings" "time" + log "github.com/Sirupsen/logrus" "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" ) const ( taskFailStatus = "error" - taskTypeID = "task" + taskTypeID = "task" ) - type task struct { task db.Task template db.Template @@ -50,7 +50,9 @@ func (t *task) prepareRun() { t.prepared = false defer func() { - fmt.Println("Stopped preparing task") + log.Info("Stopped preparing task " + strconv.Itoa(t.task.ID)) + log.Info("Release resourse locker with task " + strconv.Itoa(t.task.ID)) + resourceLocker <- &resourceLock{lock: false, holder: t} objType := taskTypeID desc := "Task ID " + strconv.Itoa(t.task.ID) + " (" + t.template.Alias + ")" + " finished - " + strings.ToUpper(t.task.Status) @@ -130,7 +132,8 @@ func (t *task) prepareRun() { func (t *task) run() { defer func() { - fmt.Println("Stopped running tasks") + log.Info("Stopped running task " + strconv.Itoa(t.task.ID)) + log.Info("Release resourse locker with task " + strconv.Itoa(t.task.ID)) resourceLocker <- &resourceLock{lock: false, holder: t} now := time.Now() @@ -299,7 +302,7 @@ func (t *task) updateRepository() error { repoName := "repository_" + strconv.Itoa(t.repository.ID) _, err := os.Stat(util.Config.TmpPath + "/" + repoName) - cmd := exec.Command("git")//nolint: gas + cmd := exec.Command("git") //nolint: gas cmd.Dir = util.Config.TmpPath gitSSHCommand := "ssh -o StrictHostKeyChecking=no -i " + t.repository.SSHKey.GetPath() @@ -335,7 +338,7 @@ func (t *task) runGalaxy() error { "--force", } - cmd := exec.Command("ansible-galaxy", args...)//nolint: gas + cmd := exec.Command(util.Config.AnsibleGalaxyCommand, args...) //nolint: gas cmd.Dir = util.Config.TmpPath + "/repository_" + strconv.Itoa(t.repository.ID) gitSSHCommand := "ssh -o StrictHostKeyChecking=no -i " + t.repository.SSHKey.GetPath() @@ -350,13 +353,18 @@ func (t *task) runGalaxy() error { } func (t *task) listPlaybookHosts() (string, error) { + + if util.Config.ConcurrencyMode == "project" { + return "", nil + } + args, err := t.getPlaybookArgs() if err != nil { return "", err } args = append(args, "--list-hosts") - cmd := exec.Command("ansible-playbook", args...)//nolint: gas + cmd := exec.Command(util.Config.AnsiblePlaybookCommand, args...) //nolint: gas cmd.Dir = util.Config.TmpPath + "/repository_" + strconv.Itoa(t.repository.ID) cmd.Env = t.envVars(util.Config.TmpPath, cmd.Dir, nil) @@ -380,7 +388,7 @@ func (t *task) runPlaybook() error { if err != nil { return err } - cmd := exec.Command("ansible-playbook", args...)//nolint: gas + cmd := exec.Command(util.Config.AnsiblePlaybookCommand, args...) //nolint: gas cmd.Dir = util.Config.TmpPath + "/repository_" + strconv.Itoa(t.repository.ID) cmd.Env = t.envVars(util.Config.TmpPath, cmd.Dir, nil) @@ -436,9 +444,18 @@ func (t *task) getPlaybookArgs() ([]string, error) { args = append(args, "--extra-vars", extraVar) } - var extraArgs []string + var templateExtraArgs []string if t.template.Arguments != nil { - err := json.Unmarshal([]byte(*t.template.Arguments), &extraArgs) + err := json.Unmarshal([]byte(*t.template.Arguments), &templateExtraArgs) + if err != nil { + t.log("Could not unmarshal arguments to []string") + return nil, err + } + } + + var taskExtraArgs []string + if t.task.Arguments != nil { + err := json.Unmarshal([]byte(*t.task.Arguments), &taskExtraArgs) if err != nil { t.log("Could not unmarshal arguments to []string") return nil, err @@ -446,9 +463,10 @@ func (t *task) getPlaybookArgs() ([]string, error) { } if t.template.OverrideArguments { - args = extraArgs + args = templateExtraArgs } else { - args = append(args, extraArgs...) + args = append(args, templateExtraArgs...) + args = append(args, taskExtraArgs...) args = append(args, playbookName) } return args, nil diff --git a/api/user.go b/api/user.go index a2bad4c11..daace3ba3 100644 --- a/api/user.go +++ b/api/user.go @@ -9,7 +9,7 @@ import ( "time" "github.com/ansible-semaphore/semaphore/db" - "github.com/castawaylabs/mulekick" + "github.com/strangeman/mulekick" "github.com/gorilla/context" "github.com/gorilla/mux" ) diff --git a/api/users.go b/api/users.go index da4a62861..01ae7c3d8 100644 --- a/api/users.go +++ b/api/users.go @@ -8,7 +8,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" + "github.com/strangeman/mulekick" "github.com/gorilla/context" "golang.org/x/crypto/bcrypt" ) diff --git a/cli/main.go b/cli/main.go index 79b034c9a..b703a0c5f 100644 --- a/cli/main.go +++ b/cli/main.go @@ -36,6 +36,7 @@ func main() { } fmt.Printf("Semaphore %v\n", util.Version) + fmt.Printf("Interface %v\n", util.Config.Interface) fmt.Printf("Port %v\n", util.Config.Port) fmt.Printf("MySQL %v@%v %v\n", util.Config.MySQL.Username, util.Config.MySQL.Hostname, util.Config.MySQL.DbName) fmt.Printf("Tmp Path (projects home) %v\n", util.Config.TmpPath) @@ -64,7 +65,7 @@ func main() { var router http.Handler = api.Route() router = handlers.ProxyHeaders(router) http.Handle("/", router) - err := http.ListenAndServe(util.Config.Port, nil) + err := http.ListenAndServe(util.Config.Interface+util.Config.Port, nil) if err != nil { log.Panic(err) } diff --git a/db/Task.go b/db/Task.go index 661edda2c..46ee84a4c 100644 --- a/db/Task.go +++ b/db/Task.go @@ -2,7 +2,6 @@ package db import "time" - //Task is a model of a task which will be executed by the runner type Task struct { ID int `db:"id" json:"id"` @@ -16,6 +15,8 @@ type Task struct { // override variables Playbook string `db:"playbook" json:"playbook"` Environment string `db:"environment" json:"environment"` + // to fit into []string + Arguments *string `db:"arguments" json:"arguments"` UserID *int `db:"user_id" json:"user_id"` diff --git a/db/Template.go b/db/Template.go index 613e0ee49..35feb9167 100644 --- a/db/Template.go +++ b/db/Template.go @@ -2,7 +2,8 @@ package db // Template is a user defined model that is used to run a task type Template struct { - ID int `db:"id" json:"id"` + ID int `db:"id" json:"id"` + Slug string `db:"slug" json:"slug"` SSHKeyID int `db:"ssh_key_id" json:"ssh_key_id"` ProjectID int `db:"project_id" json:"project_id"` diff --git a/db/migrations/v2.5.2.sql b/db/migrations/v2.5.2.sql new file mode 100644 index 000000000..616e3740f --- /dev/null +++ b/db/migrations/v2.5.2.sql @@ -0,0 +1,4 @@ +ALTER TABLE task add `arguments` text null; +ALTER TABLE `project__template` ADD slug varchar(64) not null default 'someslug' after id ; +UPDATE `project__template` set slug = CONCAT('slug-', CAST(id AS CHAR)); +ALTER TABLE `project__template` ADD UNIQUE (slug); diff --git a/db/version.go b/db/version.go index 00c7facf4..497be6b79 100644 --- a/db/version.go +++ b/db/version.go @@ -82,5 +82,6 @@ func init() { {Major: 2, Minor: 3, Patch: 2}, {Major: 2, Minor: 4}, {Major: 2, Minor: 5}, + {Major: 2, Minor: 5, Patch: 2}, } } diff --git a/deployment/docker/ci/docker-compose.yml b/deployment/docker/ci/docker-compose.yml index 7626975fa..32cee6420 100644 --- a/deployment/docker/ci/docker-compose.yml +++ b/deployment/docker/ci/docker-compose.yml @@ -8,6 +8,9 @@ services: MYSQL_DATABASE: semaphore MYSQL_USER: semaphore MYSQL_PASSWORD: semaphore + ## uncomment if you want to store mysql data between launches + #volumes: + # - /tmp/mysql_data:/var/lib/mysql ports: - "3306:3306" diff --git a/deployment/docker/dev/Dockerfile b/deployment/docker/dev/Dockerfile index c51287694..92d33bd29 100644 --- a/deployment/docker/dev/Dockerfile +++ b/deployment/docker/dev/Dockerfile @@ -22,13 +22,15 @@ RUN apk add --no-cache git mysql-client python py-pip py-openssl openssl ca-cert chown -R semaphore:0 /etc/semaphore && \ ssh-keygen -t rsa -q -f "/root/.ssh/id_rsa" -N "" && \ ssh-keyscan -H github.com > /root/.ssh/known_hosts && \ - go get -u -v github.com/go-task/task/cmd/task + curl -sL https://taskfile.dev/install.sh | sh && \ + mv ./bin/task /usr/bin/task # Copy in app source WORKDIR ${APP_ROOT} COPY . ${APP_ROOT} RUN deployment/docker/dev/bin/install -USER 1002 +USER root EXPOSE 3000 + CMD ["task", "watch"] \ No newline at end of file diff --git a/deployment/docker/dev/bin/install b/deployment/docker/dev/bin/install index 3c438b979..40e68fe6a 100755 --- a/deployment/docker/dev/bin/install +++ b/deployment/docker/dev/bin/install @@ -11,4 +11,5 @@ EOF echo "--> Install Semaphore entrypoint wrapper script" mv ./deployment/docker/common/semaphore-wrapper /usr/local/bin/semaphore-wrapper -task deps \ No newline at end of file +task deps +chmod -R 0777 /go \ No newline at end of file diff --git a/deployment/docker/dev/docker-compose-dredd.yml b/deployment/docker/dev/docker-compose-dredd.yml new file mode 100644 index 000000000..c1d6ac23e --- /dev/null +++ b/deployment/docker/dev/docker-compose-dredd.yml @@ -0,0 +1,11 @@ +version: '2' + +services: + + dredd: + image: ansiblesemaphore/dredd:dev-compose + command: ["--config", ".dredd/dredd.local.yml"] + build: + context: ./../../../ + dockerfile: ./deployment/docker/dev/dredd.Dockerfile + network_mode: "host" diff --git a/deployment/docker/dev/docker-compose.yml b/deployment/docker/dev/docker-compose.yml index cf641bf61..8b67e9aa0 100644 --- a/deployment/docker/dev/docker-compose.yml +++ b/deployment/docker/dev/docker-compose.yml @@ -8,6 +8,9 @@ services: MYSQL_DATABASE: semaphore MYSQL_USER: semaphore MYSQL_PASSWORD: semaphore + ## uncomment if you want to store mysql data between launches + #volumes: + # - /tmp/mysql_data:/var/lib/mysql ports: - "3306:3306" @@ -34,4 +37,3 @@ services: - "3000:3000" depends_on: - mysql - diff --git a/deployment/docker/dev/dredd.Dockerfile b/deployment/docker/dev/dredd.Dockerfile new file mode 100644 index 000000000..ab209762d --- /dev/null +++ b/deployment/docker/dev/dredd.Dockerfile @@ -0,0 +1,24 @@ +# hadolint ignore=DL3006 +FROM tomwhiston/dredd:latest + +ENV TASK_VERSION=v2.0.1 \ + GOPATH=/home/developer/go \ + SEMAPHORE_SERVICE=127.0.0.1 \ + SEMAPHORE_PORT=3000 \ + MYSQL_SERVICE=127.0.0.1 \ + MYSQL_PORT=3306 + +# We need the source and task to compile the hooks +USER 0 +RUN dnf install -y nc +COPY deployment/docker/ci/dredd/entrypoint /usr/local/bin +COPY . /home/developer/go/src/github.com/ansible-semaphore/semaphore +WORKDIR /usr/local/bin +RUN curl -L "https://github.com/go-task/task/releases/download/${TASK_VERSION}/task_linux_amd64.tar.gz" | tar xvz && \ + chown -R developer /home/developer/go + +# Get tools and do compile +WORKDIR /home/developer/go/src/github.com/ansible-semaphore/semaphore +RUN task deps:tools && task deps:be && task compile:be && task compile:api:hooks + +ENTRYPOINT ["/usr/local/bin/entrypoint"] \ No newline at end of file diff --git a/public/vendor b/public/vendor new file mode 160000 index 000000000..31cbac89f --- /dev/null +++ b/public/vendor @@ -0,0 +1 @@ +Subproject commit 31cbac89f1465de60797d2879addc8c71de93d1b diff --git a/util/config.go b/util/config.go index 2f5bfcced..7af369813 100644 --- a/util/config.go +++ b/util/config.go @@ -49,11 +49,20 @@ type ldapMappings struct { type configType struct { MySQL mySQLConfig `json:"mysql"` // Format `:port_num` eg, :3000 + // if : is missing it will be corrected Port string `json:"port"` + // Interface ip, put in front of the port. + // defaults to empty + Interface string `json:"interface"` + // semaphore stores ephemeral projects here TmpPath string `json:"tmp_path"` + // useful for overriding default ansible binaries + AnsibleGalaxyCommand string `json:"ansible_galaxy_command"` + AnsiblePlaybookCommand string `json:"ansible_playbook_command"` + // cookie hashing & encryption CookieHash string `json:"cookie_hash"` CookieEncryption string `json:"cookie_encryption"` @@ -200,6 +209,14 @@ func validateConfig() { Config.TmpPath = "/tmp/semaphore" } + if len(Config.AnsibleGalaxyCommand) == 0 { + Config.AnsibleGalaxyCommand = "ansible-galaxy" + } + + if len(Config.AnsiblePlaybookCommand) == 0 { + Config.AnsiblePlaybookCommand = "ansible-playbook" + } + if Config.MaxParallelTasks < 1 { Config.MaxParallelTasks = 10 } diff --git a/web/resources/pug/projects/createTaskModal.pug b/web/resources/pug/projects/createTaskModal.pug index 0c445e25e..56c92da69 100644 --- a/web/resources/pug/projects/createTaskModal.pug +++ b/web/resources/pug/projects/createTaskModal.pug @@ -13,6 +13,10 @@ label.control-label.col-sm-4 Environment Override (*MUST* be valid JSON) .col-sm-6 div(ui-ace="{mode: 'json', workerPath: '/public/js/ace/'}" class="form-control" style="height: 100px" ng-model="task.environment") + .form-group + label.control-label.col-sm-4(uib-tooltip='*MUST* be a JSON array! Each argument must be an element of the array, for example: ["-i", "@myinventory.sh", "--private-key=/there/id_rsa", "-vvvv"]') Extra CLI Arguments + .col-sm-6 + div(ui-ace="{mode: 'json', workerPath: '/public/js/ace/'}" style="height: 100px" class="form-control" ng-model="task.arguments") .form-group .col-sm-6.col-sm-offset-4: .checkbox: label input(type="checkbox" ng-model="task.debug") diff --git a/web/resources/pug/projects/templates/add.pug b/web/resources/pug/projects/templates/add.pug index 29f29457d..501505822 100644 --- a/web/resources/pug/projects/templates/add.pug +++ b/web/resources/pug/projects/templates/add.pug @@ -14,6 +14,11 @@ .col-sm-6 input.form-control(type="text" readonly="readonly" ng-model="tpl.id") + .form-group + label.control-label.col-sm-4 Template Slug + .col-sm-6 + input.form-control(type="text" placeholder="some-task" ng-model="tpl.slug") + .form-group label.control-label.col-sm-4 Playbook Name .col-sm-6 diff --git a/web/resources/pug/projects/templates/list.pug b/web/resources/pug/projects/templates/list.pug index cbef65bef..bc1496303 100644 --- a/web/resources/pug/projects/templates/list.pug +++ b/web/resources/pug/projects/templates/list.pug @@ -6,6 +6,7 @@ h3 Task Templates table.table.table-hover thead: tr th Alias + th Slug th Playbook th SSH Key th Inventory @@ -14,6 +15,7 @@ table.table.table-hover th   tbody: tr(ng-repeat="tpl in templates" ng-click="update(tpl)" style="cursor: pointer;" ng-if="!tpl.hidden || allShown") td {{ tpl.alias }} + td {{ tpl.slug }} td {{ tpl.playbook }} td {{ sshKeysAssoc[tpl.ssh_key_id].name }} td {{ inventoryAssoc[tpl.inventory_id].name }}