From 373f3d1e8e1f71843c9cb18464147fc8464389da Mon Sep 17 00:00:00 2001 From: tom whiston Date: Mon, 14 May 2018 19:37:07 +0000 Subject: [PATCH 01/20] add interface variable to settings --- cli/main.go | 3 ++- util/config.go | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cli/main.go b/cli/main.go index 79b034c9a..b703a0c5f 100644 --- a/cli/main.go +++ b/cli/main.go @@ -36,6 +36,7 @@ func main() { } fmt.Printf("Semaphore %v\n", util.Version) + fmt.Printf("Interface %v\n", util.Config.Interface) fmt.Printf("Port %v\n", util.Config.Port) fmt.Printf("MySQL %v@%v %v\n", util.Config.MySQL.Username, util.Config.MySQL.Hostname, util.Config.MySQL.DbName) fmt.Printf("Tmp Path (projects home) %v\n", util.Config.TmpPath) @@ -64,7 +65,7 @@ func main() { var router http.Handler = api.Route() router = handlers.ProxyHeaders(router) http.Handle("/", router) - err := http.ListenAndServe(util.Config.Port, nil) + err := http.ListenAndServe(util.Config.Interface+util.Config.Port, nil) if err != nil { log.Panic(err) } diff --git a/util/config.go b/util/config.go index 2f5bfcced..d3fa5174a 100644 --- a/util/config.go +++ b/util/config.go @@ -49,8 +49,13 @@ type ldapMappings struct { type configType struct { MySQL mySQLConfig `json:"mysql"` // Format `:port_num` eg, :3000 + // if : is missing it will be corrected Port string `json:"port"` + // Interface ip, put in front of the port. + // defaults to empty + Interface string `json:"interface"` + // semaphore stores ephemeral projects here TmpPath string `json:"tmp_path"` From 087acb8ed4ca3db6aad727905793c049259c7cf0 Mon Sep 17 00:00:00 2001 From: Anton Markelov Date: Thu, 7 Jun 2018 17:29:55 +1000 Subject: [PATCH 02/20] remove collisions on prepareTask stage --- api/tasks/pool.go | 18 +++++++++++------- api/tasks/runner.go | 17 +++++++++++------ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/api/tasks/pool.go b/api/tasks/pool.go index 61e9bf80a..4ea92a35e 100644 --- a/api/tasks/pool.go +++ b/api/tasks/pool.go @@ -75,22 +75,26 @@ func (p *taskPool) run() { select { case task := <-p.register: fmt.Println(task) - go task.prepareRun() p.queue = append(p.queue, task) case <-ticker.C: if len(p.queue) == 0 { continue - } else if t := p.queue[0]; t.task.Status != taskFailStatus && (!t.prepared || p.blocks(t)) { + } else if t := p.queue[0]; t.task.Status != taskFailStatus && p.blocks(t) { p.queue = append(p.queue[1:], t) continue } - if t := pool.queue[0]; t.task.Status != taskFailStatus { - fmt.Println("Running a task.") - resourceLocker <- &resourceLock{lock: true, holder: t} - go t.run() + if t := p.queue[0]; t.task.Status != taskFailStatus { + if t.prepared { + fmt.Println("Running a task.") + resourceLocker <- &resourceLock{lock: true, holder: t} + go t.run() + p.queue = p.queue[1:] + } else { + resourceLocker <- &resourceLock{lock: true, holder: t} + go t.prepareRun() + } } - pool.queue = pool.queue[1:] } } } diff --git a/api/tasks/runner.go b/api/tasks/runner.go index 339759064..0d8ce93ed 100644 --- a/api/tasks/runner.go +++ b/api/tasks/runner.go @@ -20,10 +20,9 @@ import ( const ( taskFailStatus = "error" - taskTypeID = "task" + taskTypeID = "task" ) - type task struct { task db.Task template db.Template @@ -51,6 +50,7 @@ func (t *task) prepareRun() { defer func() { fmt.Println("Stopped preparing task") + resourceLocker <- &resourceLock{lock: false, holder: t} objType := taskTypeID desc := "Task ID " + strconv.Itoa(t.task.ID) + " (" + t.template.Alias + ")" + " finished - " + strings.ToUpper(t.task.Status) @@ -299,7 +299,7 @@ func (t *task) updateRepository() error { repoName := "repository_" + strconv.Itoa(t.repository.ID) _, err := os.Stat(util.Config.TmpPath + "/" + repoName) - cmd := exec.Command("git")//nolint: gas + cmd := exec.Command("git") //nolint: gas cmd.Dir = util.Config.TmpPath gitSSHCommand := "ssh -o StrictHostKeyChecking=no -i " + t.repository.SSHKey.GetPath() @@ -335,7 +335,7 @@ func (t *task) runGalaxy() error { "--force", } - cmd := exec.Command("ansible-galaxy", args...)//nolint: gas + cmd := exec.Command("ansible-galaxy", args...) //nolint: gas cmd.Dir = util.Config.TmpPath + "/repository_" + strconv.Itoa(t.repository.ID) gitSSHCommand := "ssh -o StrictHostKeyChecking=no -i " + t.repository.SSHKey.GetPath() @@ -350,13 +350,18 @@ func (t *task) runGalaxy() error { } func (t *task) listPlaybookHosts() (string, error) { + + if util.Config.ConcurrencyMode == "project" { + return "", nil + } + args, err := t.getPlaybookArgs() if err != nil { return "", err } args = append(args, "--list-hosts") - cmd := exec.Command("ansible-playbook", args...)//nolint: gas + cmd := exec.Command("ansible-playbook", args...) //nolint: gas cmd.Dir = util.Config.TmpPath + "/repository_" + strconv.Itoa(t.repository.ID) cmd.Env = t.envVars(util.Config.TmpPath, cmd.Dir, nil) @@ -380,7 +385,7 @@ func (t *task) runPlaybook() error { if err != nil { return err } - cmd := exec.Command("ansible-playbook", args...)//nolint: gas + cmd := exec.Command("ansible-playbook", args...) //nolint: gas cmd.Dir = util.Config.TmpPath + "/repository_" + strconv.Itoa(t.repository.ID) cmd.Env = t.envVars(util.Config.TmpPath, cmd.Dir, nil) From a817d2d1274c6a2b2d122af8cf750811d2f11ceb Mon Sep 17 00:00:00 2001 From: Anton Markelov Date: Fri, 8 Jun 2018 11:58:08 +1000 Subject: [PATCH 03/20] add requested changes, add some logging --- api/tasks/pool.go | 19 ++++++++++--------- api/tasks/runner.go | 7 +++++-- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/api/tasks/pool.go b/api/tasks/pool.go index 4ea92a35e..d63b83a87 100644 --- a/api/tasks/pool.go +++ b/api/tasks/pool.go @@ -1,9 +1,10 @@ package tasks import ( - "fmt" + "strconv" "time" + log "github.com/Sirupsen/logrus" "github.com/ansible-semaphore/semaphore/util" ) @@ -74,8 +75,9 @@ func (p *taskPool) run() { for { select { case task := <-p.register: - fmt.Println(task) p.queue = append(p.queue, task) + log.Debug(task) + task.log("Task " + strconv.Itoa(task.task.ID) + " added to queue") case <-ticker.C: if len(p.queue) == 0 { continue @@ -85,15 +87,14 @@ func (p *taskPool) run() { } if t := p.queue[0]; t.task.Status != taskFailStatus { - if t.prepared { - fmt.Println("Running a task.") - resourceLocker <- &resourceLock{lock: true, holder: t} - go t.run() - p.queue = p.queue[1:] - } else { - resourceLocker <- &resourceLock{lock: true, holder: t} + log.Info("Set resourse locker with task " + strconv.Itoa(t.task.ID)) + resourceLocker <- &resourceLock{lock: true, holder: t} + if !t.prepared { go t.prepareRun() + continue } + go t.run() + p.queue = p.queue[1:] } } } diff --git a/api/tasks/runner.go b/api/tasks/runner.go index 0d8ce93ed..9eed70c21 100644 --- a/api/tasks/runner.go +++ b/api/tasks/runner.go @@ -14,6 +14,7 @@ import ( "strings" "time" + log "github.com/Sirupsen/logrus" "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" ) @@ -49,7 +50,8 @@ func (t *task) prepareRun() { t.prepared = false defer func() { - fmt.Println("Stopped preparing task") + log.Info("Stopped preparing task " + strconv.Itoa(t.task.ID)) + log.Info("Release resourse locker with task " + strconv.Itoa(t.task.ID)) resourceLocker <- &resourceLock{lock: false, holder: t} objType := taskTypeID @@ -130,7 +132,8 @@ func (t *task) prepareRun() { func (t *task) run() { defer func() { - fmt.Println("Stopped running tasks") + log.Info("Stopped running task " + strconv.Itoa(t.task.ID)) + log.Info("Release resourse locker with task " + strconv.Itoa(t.task.ID)) resourceLocker <- &resourceLock{lock: false, holder: t} now := time.Now() From b60213aa885c02657b45e2aa3d475a86f262f763 Mon Sep 17 00:00:00 2001 From: Anton Markelov Date: Fri, 8 Jun 2018 18:20:41 +1000 Subject: [PATCH 04/20] remove task from queue if it failed (on prepare step) --- api/tasks/pool.go | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/api/tasks/pool.go b/api/tasks/pool.go index d63b83a87..ebe426646 100644 --- a/api/tasks/pool.go +++ b/api/tasks/pool.go @@ -77,25 +77,36 @@ func (p *taskPool) run() { case task := <-p.register: p.queue = append(p.queue, task) log.Debug(task) - task.log("Task " + strconv.Itoa(task.task.ID) + " added to queue") + msg := "Task " + strconv.Itoa(task.task.ID) + " added to queue" + task.log(msg) + log.Info(msg) case <-ticker.C: if len(p.queue) == 0 { continue - } else if t := p.queue[0]; t.task.Status != taskFailStatus && p.blocks(t) { - p.queue = append(p.queue[1:], t) - continue } - if t := p.queue[0]; t.task.Status != taskFailStatus { - log.Info("Set resourse locker with task " + strconv.Itoa(t.task.ID)) - resourceLocker <- &resourceLock{lock: true, holder: t} - if !t.prepared { - go t.prepareRun() - continue - } - go t.run() + //get task from top of queue + t := p.queue[0] + if t.task.Status == taskFailStatus { + //delete failed task from queue p.queue = p.queue[1:] + log.Info("Task " + strconv.Itoa(t.task.ID) + " removed from queue") + continue + } + if p.blocks(t) { + //move blocked task to end of queue + p.queue = append(p.queue[1:], t) + continue + } + log.Info("Set resourse locker with task " + strconv.Itoa(t.task.ID)) + resourceLocker <- &resourceLock{lock: true, holder: t} + if !t.prepared { + go t.prepareRun() + continue } + go t.run() + p.queue = p.queue[1:] + log.Info("Task " + strconv.Itoa(t.task.ID) + " removed from queue") } } } From cd30536a85c0da6534b073795f9961ef0ec8bac0 Mon Sep 17 00:00:00 2001 From: Anton Markelov Date: Thu, 14 Jun 2018 16:20:16 +1000 Subject: [PATCH 05/20] make semaphore less panic --- api/tasks/http.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/api/tasks/http.go b/api/tasks/http.go index 8d3175c19..51cab0129 100644 --- a/api/tasks/http.go +++ b/api/tasks/http.go @@ -13,7 +13,7 @@ import ( "github.com/masterminds/squirrel" ) -// AddTask inserts a task into the database and returns a header or panics +// AddTask inserts a task into the database and returns a header or returns error func AddTask(w http.ResponseWriter, r *http.Request) { project := context.Get(r, "project").(db.Project) user := context.Get(r, "user").(*db.User) @@ -28,7 +28,9 @@ func AddTask(w http.ResponseWriter, r *http.Request) { taskObj.UserID = &user.ID if err := db.Mysql.Insert(&taskObj); err != nil { - panic(err) + util.LogErrorWithFields(err, log.Fields{"error": "Bad request. Cannot create new task"}) + w.WriteHeader(http.StatusBadRequest) + return } pool.register <- &task{ @@ -44,13 +46,13 @@ func AddTask(w http.ResponseWriter, r *http.Request) { ObjectID: &taskObj.ID, Description: &desc, }.Insert()); err != nil { - panic(err) + util.LogErrorWithFields(err, log.Fields{"error": "Cannot write new event to database"}) } mulekick.WriteJSON(w, http.StatusCreated, taskObj) } -// GetTasksList returns a list of tasks for the current project in desc order to limit or panics +// GetTasksList returns a list of tasks for the current project in desc order to limit or error func GetTasksList(w http.ResponseWriter, r *http.Request, limit uint64) { project := context.Get(r, "project").(db.Project) @@ -75,7 +77,9 @@ func GetTasksList(w http.ResponseWriter, r *http.Request, limit uint64) { UserName *string `db:"user_name" json:"user_name"` } if _, err := db.Mysql.Select(&tasks, query, args...); err != nil { - panic(err) + util.LogErrorWithFields(err, log.Fields{"error": "Bad request. Cannot get tasks list from database"}) + w.WriteHeader(http.StatusBadRequest) + return } mulekick.WriteJSON(w, http.StatusOK, tasks) @@ -112,13 +116,15 @@ func GetTaskMiddleware(w http.ResponseWriter, r *http.Request) { context.Set(r, taskTypeID, task) } -// GetTaskOutput returns the logged task output by id and writes it as json +// GetTaskOutput returns the logged task output by id and writes it as json or returns error func GetTaskOutput(w http.ResponseWriter, r *http.Request) { task := context.Get(r, taskTypeID).(db.Task) var output []db.TaskOutput if _, err := db.Mysql.Select(&output, "select task_id, task, time, output from task__output where task_id=? order by time asc", task.ID); err != nil { - panic(err) + util.LogErrorWithFields(err, log.Fields{"error": "Bad request. Cannot get task output from database"}) + w.WriteHeader(http.StatusBadRequest) + return } mulekick.WriteJSON(w, http.StatusOK, output) @@ -143,7 +149,9 @@ func RemoveTask(w http.ResponseWriter, r *http.Request) { for _, statement := range statements { _, err := db.Mysql.Exec(statement, task.ID) if err != nil { - panic(err) + util.LogErrorWithFields(err, log.Fields{"error": "Bad request. Cannot delete task from database"}) + w.WriteHeader(http.StatusBadRequest) + return } } From c5f402bfa3f1a1ec0e42d401419b80b81ee42990 Mon Sep 17 00:00:00 2001 From: dahyung-kwon Date: Tue, 11 Sep 2018 20:22:54 +0900 Subject: [PATCH 06/20] =?UTF-8?q?Add=20=E2=80=9CExtra=20CLI=20Arguments?= =?UTF-8?q?=E2=80=9D=20filed=20to=20"Create=20Task"=20UI.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/resources/pug/projects/createTaskModal.pug | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web/resources/pug/projects/createTaskModal.pug b/web/resources/pug/projects/createTaskModal.pug index 0c445e25e..56c92da69 100644 --- a/web/resources/pug/projects/createTaskModal.pug +++ b/web/resources/pug/projects/createTaskModal.pug @@ -13,6 +13,10 @@ label.control-label.col-sm-4 Environment Override (*MUST* be valid JSON) .col-sm-6 div(ui-ace="{mode: 'json', workerPath: '/public/js/ace/'}" class="form-control" style="height: 100px" ng-model="task.environment") + .form-group + label.control-label.col-sm-4(uib-tooltip='*MUST* be a JSON array! Each argument must be an element of the array, for example: ["-i", "@myinventory.sh", "--private-key=/there/id_rsa", "-vvvv"]') Extra CLI Arguments + .col-sm-6 + div(ui-ace="{mode: 'json', workerPath: '/public/js/ace/'}" style="height: 100px" class="form-control" ng-model="task.arguments") .form-group .col-sm-6.col-sm-offset-4: .checkbox: label input(type="checkbox" ng-model="task.debug") From 4814b7ee40cf4d7808c75ff66c0775ab5d0a0878 Mon Sep 17 00:00:00 2001 From: dahyung-kwon Date: Tue, 11 Sep 2018 20:29:05 +0900 Subject: [PATCH 07/20] Add arguments column on task table. --- db/Task.go | 3 ++- db/migrations/v2.5.2.sql | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 db/migrations/v2.5.2.sql diff --git a/db/Task.go b/db/Task.go index 661edda2c..46ee84a4c 100644 --- a/db/Task.go +++ b/db/Task.go @@ -2,7 +2,6 @@ package db import "time" - //Task is a model of a task which will be executed by the runner type Task struct { ID int `db:"id" json:"id"` @@ -16,6 +15,8 @@ type Task struct { // override variables Playbook string `db:"playbook" json:"playbook"` Environment string `db:"environment" json:"environment"` + // to fit into []string + Arguments *string `db:"arguments" json:"arguments"` UserID *int `db:"user_id" json:"user_id"` diff --git a/db/migrations/v2.5.2.sql b/db/migrations/v2.5.2.sql new file mode 100644 index 000000000..ac61adc89 --- /dev/null +++ b/db/migrations/v2.5.2.sql @@ -0,0 +1 @@ +alter table task add `arguments` text null; \ No newline at end of file From 5d1ac1241e5794187930defdb4a3dc4218b36c03 Mon Sep 17 00:00:00 2001 From: dahyung-kwon Date: Tue, 11 Sep 2018 20:49:03 +0900 Subject: [PATCH 08/20] Add task.arguments value to getPlaybookArgs() method. --- api/tasks/runner.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/api/tasks/runner.go b/api/tasks/runner.go index 9eed70c21..cdfff364d 100644 --- a/api/tasks/runner.go +++ b/api/tasks/runner.go @@ -444,9 +444,18 @@ func (t *task) getPlaybookArgs() ([]string, error) { args = append(args, "--extra-vars", extraVar) } - var extraArgs []string + var templateExtraArgs []string if t.template.Arguments != nil { - err := json.Unmarshal([]byte(*t.template.Arguments), &extraArgs) + err := json.Unmarshal([]byte(*t.template.Arguments), &templateExtraArgs) + if err != nil { + t.log("Could not unmarshal arguments to []string") + return nil, err + } + } + + var taskExtraArgs []string + if t.task.Arguments != nil { + err := json.Unmarshal([]byte(*t.task.Arguments), &taskExtraArgs) if err != nil { t.log("Could not unmarshal arguments to []string") return nil, err @@ -454,9 +463,10 @@ func (t *task) getPlaybookArgs() ([]string, error) { } if t.template.OverrideArguments { - args = extraArgs + args = templateExtraArgs } else { - args = append(args, extraArgs...) + args = append(args, templateExtraArgs...) + args = append(args, taskExtraArgs...) args = append(args, playbookName) } return args, nil From 70bea1e1334efda7bf3c64c81bc61126d8306f6d Mon Sep 17 00:00:00 2001 From: "dahyung.kwon" Date: Tue, 16 Oct 2018 00:48:19 +0900 Subject: [PATCH 09/20] add version 2.5.2 --- db/version.go | 1 + 1 file changed, 1 insertion(+) diff --git a/db/version.go b/db/version.go index 00c7facf4..497be6b79 100644 --- a/db/version.go +++ b/db/version.go @@ -82,5 +82,6 @@ func init() { {Major: 2, Minor: 3, Patch: 2}, {Major: 2, Minor: 4}, {Major: 2, Minor: 5}, + {Major: 2, Minor: 5, Patch: 2}, } } From 445053442398f9b4348fbb35b16ebec7cc855db0 Mon Sep 17 00:00:00 2001 From: Anton Markelov Date: Sat, 20 Oct 2018 16:17:08 +1000 Subject: [PATCH 10/20] documentation hotfix --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 900578b5d..2c8e52e1e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,7 +96,7 @@ This means that you should never run these tests against your productive databas The best practice to run these tests is to use docker and the task commands. ```bash -task dc:build #First run only to build the images +context=dev task dc:build #First run only to build the images context=dev task dc:up task test:api # alternatively if you want to run dredd in a container use the following command. From cbe0c5ffc88c81a66d229b5643e114c257851459 Mon Sep 17 00:00:00 2001 From: Anton Markelov Date: Sat, 20 Oct 2018 21:56:32 +1000 Subject: [PATCH 11/20] add compiled_hooks to gitignore --- .gitignore | 2 ++ deployment/docker/dev/Dockerfile | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 559d55941..e0203585e 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ util/version.go /coverage.out /public/package-lock.json !.gitkeep + +.dredd/compiled_hooks \ No newline at end of file diff --git a/deployment/docker/dev/Dockerfile b/deployment/docker/dev/Dockerfile index c51287694..3bc823346 100644 --- a/deployment/docker/dev/Dockerfile +++ b/deployment/docker/dev/Dockerfile @@ -29,6 +29,7 @@ WORKDIR ${APP_ROOT} COPY . ${APP_ROOT} RUN deployment/docker/dev/bin/install -USER 1002 +USER 1000 EXPOSE 3000 + CMD ["task", "watch"] \ No newline at end of file From 7bab8e03322003e074d460780088f999b5a65eb2 Mon Sep 17 00:00:00 2001 From: Anton Markelov Date: Mon, 22 Oct 2018 12:13:59 +1000 Subject: [PATCH 12/20] add dredd to dev dc environment --- deployment/docker/ci/docker-compose.yml | 3 +++ .../docker/dev/docker-compose-dredd.yml | 12 ++++++++++ deployment/docker/dev/docker-compose.yml | 4 +++- deployment/docker/dev/dredd.Dockerfile | 24 +++++++++++++++++++ public/vendor | 1 + 5 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 deployment/docker/dev/docker-compose-dredd.yml create mode 100644 deployment/docker/dev/dredd.Dockerfile create mode 160000 public/vendor diff --git a/deployment/docker/ci/docker-compose.yml b/deployment/docker/ci/docker-compose.yml index 7626975fa..32cee6420 100644 --- a/deployment/docker/ci/docker-compose.yml +++ b/deployment/docker/ci/docker-compose.yml @@ -8,6 +8,9 @@ services: MYSQL_DATABASE: semaphore MYSQL_USER: semaphore MYSQL_PASSWORD: semaphore + ## uncomment if you want to store mysql data between launches + #volumes: + # - /tmp/mysql_data:/var/lib/mysql ports: - "3306:3306" diff --git a/deployment/docker/dev/docker-compose-dredd.yml b/deployment/docker/dev/docker-compose-dredd.yml new file mode 100644 index 000000000..1fade045d --- /dev/null +++ b/deployment/docker/dev/docker-compose-dredd.yml @@ -0,0 +1,12 @@ +version: '2' + +services: + + dredd: + image: ansiblesemaphore/dredd:dev-compose + command: ["--config", ".dredd/dredd.yml"] + build: + context: ./../../../ + dockerfile: ./deployment/docker/dev/dredd.Dockerfile + external_links: + - dev_semaphore_dev_1:semaphore_dev \ No newline at end of file diff --git a/deployment/docker/dev/docker-compose.yml b/deployment/docker/dev/docker-compose.yml index cf641bf61..8b67e9aa0 100644 --- a/deployment/docker/dev/docker-compose.yml +++ b/deployment/docker/dev/docker-compose.yml @@ -8,6 +8,9 @@ services: MYSQL_DATABASE: semaphore MYSQL_USER: semaphore MYSQL_PASSWORD: semaphore + ## uncomment if you want to store mysql data between launches + #volumes: + # - /tmp/mysql_data:/var/lib/mysql ports: - "3306:3306" @@ -34,4 +37,3 @@ services: - "3000:3000" depends_on: - mysql - diff --git a/deployment/docker/dev/dredd.Dockerfile b/deployment/docker/dev/dredd.Dockerfile new file mode 100644 index 000000000..705441749 --- /dev/null +++ b/deployment/docker/dev/dredd.Dockerfile @@ -0,0 +1,24 @@ +# hadolint ignore=DL3006 +FROM tomwhiston/dredd:latest + +ENV TASK_VERSION=v2.0.1 \ + GOPATH=/home/developer/go \ + SEMAPHORE_SERVICE=semaphore_dev \ + SEMAPHORE_PORT=3000 \ + MYSQL_SERVICE=mysql \ + MYSQL_PORT=3306 + +# We need the source and task to compile the hooks +USER 0 +RUN dnf install -y nc +COPY deployment/docker/ci/dredd/entrypoint /usr/local/bin +COPY . /home/developer/go/src/github.com/ansible-semaphore/semaphore +WORKDIR /usr/local/bin +RUN curl -L "https://github.com/go-task/task/releases/download/${TASK_VERSION}/task_linux_amd64.tar.gz" | tar xvz && \ + chown -R developer /home/developer/go + +# Get tools and do compile +WORKDIR /home/developer/go/src/github.com/ansible-semaphore/semaphore +RUN task deps:tools && task deps:be && task compile:be && task compile:api:hooks + +ENTRYPOINT ["/usr/local/bin/entrypoint"] \ No newline at end of file diff --git a/public/vendor b/public/vendor new file mode 160000 index 000000000..31cbac89f --- /dev/null +++ b/public/vendor @@ -0,0 +1 @@ +Subproject commit 31cbac89f1465de60797d2879addc8c71de93d1b From bdced30dac8d832b87a14ec9fd3bbaaa24fcee57 Mon Sep 17 00:00:00 2001 From: Anton Markelov Date: Mon, 22 Oct 2018 12:59:02 +1000 Subject: [PATCH 13/20] fix dredd build and launch --- deployment/docker/dev/docker-compose-dredd.yml | 5 ++--- deployment/docker/dev/dredd.Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/deployment/docker/dev/docker-compose-dredd.yml b/deployment/docker/dev/docker-compose-dredd.yml index 1fade045d..c1d6ac23e 100644 --- a/deployment/docker/dev/docker-compose-dredd.yml +++ b/deployment/docker/dev/docker-compose-dredd.yml @@ -4,9 +4,8 @@ services: dredd: image: ansiblesemaphore/dredd:dev-compose - command: ["--config", ".dredd/dredd.yml"] + command: ["--config", ".dredd/dredd.local.yml"] build: context: ./../../../ dockerfile: ./deployment/docker/dev/dredd.Dockerfile - external_links: - - dev_semaphore_dev_1:semaphore_dev \ No newline at end of file + network_mode: "host" diff --git a/deployment/docker/dev/dredd.Dockerfile b/deployment/docker/dev/dredd.Dockerfile index 705441749..ab209762d 100644 --- a/deployment/docker/dev/dredd.Dockerfile +++ b/deployment/docker/dev/dredd.Dockerfile @@ -3,9 +3,9 @@ FROM tomwhiston/dredd:latest ENV TASK_VERSION=v2.0.1 \ GOPATH=/home/developer/go \ - SEMAPHORE_SERVICE=semaphore_dev \ + SEMAPHORE_SERVICE=127.0.0.1 \ SEMAPHORE_PORT=3000 \ - MYSQL_SERVICE=mysql \ + MYSQL_SERVICE=127.0.0.1 \ MYSQL_PORT=3306 # We need the source and task to compile the hooks From 1bb7fc5afe92950bba6a348608460b46e3cb7481 Mon Sep 17 00:00:00 2001 From: Anton Markelov Date: Mon, 22 Oct 2018 13:33:28 +1000 Subject: [PATCH 14/20] fix `permission denied` errors --- deployment/docker/dev/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/docker/dev/docker-compose.yml b/deployment/docker/dev/docker-compose.yml index 8b67e9aa0..59b62ea33 100644 --- a/deployment/docker/dev/docker-compose.yml +++ b/deployment/docker/dev/docker-compose.yml @@ -9,8 +9,8 @@ services: MYSQL_USER: semaphore MYSQL_PASSWORD: semaphore ## uncomment if you want to store mysql data between launches - #volumes: - # - /tmp/mysql_data:/var/lib/mysql + volumes: + - /tmp/mysql_data:/var/lib/mysql ports: - "3306:3306" From 5e71ce68b564f383df2380513de2283c957608d9 Mon Sep 17 00:00:00 2001 From: Anton Markelov Date: Mon, 22 Oct 2018 13:45:35 +1000 Subject: [PATCH 15/20] add dc dredd to taskfile --- Taskfile.yml | 23 ++++++++++++++++++++++- deployment/docker/dev/bin/install | 3 ++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index aac43dbfa..3d5eb8f63 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -241,6 +241,27 @@ tasks: args: stop context: "{{ .context }}" + dc:build:dredd: + desc: build a dredd container to the local testing with docker-compose stack + cmds: + - task: docker + vars: + context: dev + compose: true + prefix: -dredd + args: build + + dc:up:dredd: + desc: build a dredd container to the local testing with docker-compose stack + cmds: + - task: docker + vars: + context: dev + compose: true + prefix: -dredd + args: up + + docker:build: desc: Build an image for Semaphore, requires context vars: @@ -302,4 +323,4 @@ tasks: vars: docker_root: deployment/docker/ cmds: - - docker{{ if .compose }}-compose{{ end }} {{ if .action }}{{ .action }}{{ end }} -f {{ .docker_root }}{{ .context }}/{{ if .compose }}docker-compose.yml{{ else }}Dockerfile{{ end }} {{if .args }}{{ .args }}{{ end }} + - docker{{ if .compose }}-compose{{ end }} {{ if .action }}{{ .action }}{{ end }} -f {{ .docker_root }}{{ .context }}/{{ if .compose }}docker-compose{{ if .prefix }}{{ .prefix }}{{ end }}.yml{{ else }}Dockerfile{{ if .prefix }}{{ .prefix }}{{ end }}{{ end }} {{if .args }}{{ .args }}{{ end }} diff --git a/deployment/docker/dev/bin/install b/deployment/docker/dev/bin/install index 3c438b979..40e68fe6a 100755 --- a/deployment/docker/dev/bin/install +++ b/deployment/docker/dev/bin/install @@ -11,4 +11,5 @@ EOF echo "--> Install Semaphore entrypoint wrapper script" mv ./deployment/docker/common/semaphore-wrapper /usr/local/bin/semaphore-wrapper -task deps \ No newline at end of file +task deps +chmod -R 0777 /go \ No newline at end of file From d113a3ad20e9ce1c38507b1efdfbe5e8c68f8fad Mon Sep 17 00:00:00 2001 From: Anton Markelov Date: Mon, 22 Oct 2018 13:56:53 +1000 Subject: [PATCH 16/20] add documentation about dredd in docker-compose --- CONTRIBUTING.md | 10 ++++------ deployment/docker/dev/docker-compose.yml | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2c8e52e1e..1e98fe7ef 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,10 +96,8 @@ This means that you should never run these tests against your productive databas The best practice to run these tests is to use docker and the task commands. ```bash -context=dev task dc:build #First run only to build the images -context=dev task dc:up -task test:api -# alternatively if you want to run dredd in a container use the following command. -# You will need to use the host network so that it can reach the docker container on a 0.0.0.0 address -# docker run -it --rm -v $(pwd):/home/developer/src --network host tomwhiston/dredd --config .dredd/dredd.yml +context=dev task dc:build #build fresh semaphore images +context=dev task dc:up #up semaphore and mysql +task dc:build:dredd #build fresh dredd image +task dc:up:dredd #run dredd over docker-compose stack ``` \ No newline at end of file diff --git a/deployment/docker/dev/docker-compose.yml b/deployment/docker/dev/docker-compose.yml index 59b62ea33..8b67e9aa0 100644 --- a/deployment/docker/dev/docker-compose.yml +++ b/deployment/docker/dev/docker-compose.yml @@ -9,8 +9,8 @@ services: MYSQL_USER: semaphore MYSQL_PASSWORD: semaphore ## uncomment if you want to store mysql data between launches - volumes: - - /tmp/mysql_data:/var/lib/mysql + #volumes: + # - /tmp/mysql_data:/var/lib/mysql ports: - "3306:3306" From 14f5f6bb35b8a2f25c5d4542f9bc96dd03244e6a Mon Sep 17 00:00:00 2001 From: Anton Markelov Date: Mon, 22 Oct 2018 17:19:33 +1000 Subject: [PATCH 17/20] return 401 instead of 403 on unauthorized pages --- api-docs.yml | 2 +- api/auth.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api-docs.yml b/api-docs.yml index 43d42793b..dc18b6d8d 100644 --- a/api-docs.yml +++ b/api-docs.yml @@ -451,7 +451,7 @@ paths: responses: 200: description: OK - 403: + 401: description: not authenticated /info: diff --git a/api/auth.go b/api/auth.go index 3f8bc24eb..ee42aabde 100644 --- a/api/auth.go +++ b/api/auth.go @@ -20,7 +20,7 @@ func authentication(w http.ResponseWriter, r *http.Request) { var token db.APIToken if err := db.Mysql.SelectOne(&token, "select * from user__token where id=? and expired=0", strings.Replace(authHeader, "bearer ", "", 1)); err != nil { if err == sql.ErrNoRows { - w.WriteHeader(http.StatusForbidden) + w.WriteHeader(http.StatusUnauthorized) return } @@ -32,20 +32,20 @@ func authentication(w http.ResponseWriter, r *http.Request) { // fetch session from cookie cookie, err := r.Cookie("semaphore") if err != nil { - w.WriteHeader(http.StatusForbidden) + w.WriteHeader(http.StatusUnauthorized) return } value := make(map[string]interface{}) if err = util.Cookie.Decode("semaphore", cookie.Value, &value); err != nil { - w.WriteHeader(http.StatusForbidden) + w.WriteHeader(http.StatusUnauthorized) return } user, ok := value["user"] sessionVal, okSession := value["session"] if !ok || !okSession { - w.WriteHeader(http.StatusForbidden) + w.WriteHeader(http.StatusUnauthorized) return } @@ -55,7 +55,7 @@ func authentication(w http.ResponseWriter, r *http.Request) { // fetch session var session db.Session if err := db.Mysql.SelectOne(&session, "select * from session where id=? and user_id=? and expired=0", sessionID, userID); err != nil { - w.WriteHeader(http.StatusForbidden) + w.WriteHeader(http.StatusUnauthorized) return } @@ -66,7 +66,7 @@ func authentication(w http.ResponseWriter, r *http.Request) { panic(err) } - w.WriteHeader(http.StatusForbidden) + w.WriteHeader(http.StatusUnauthorized) return } @@ -78,7 +78,7 @@ func authentication(w http.ResponseWriter, r *http.Request) { user, err := db.FetchUser(userID) if err != nil { fmt.Println("Can't find user", err) - w.WriteHeader(http.StatusForbidden) + w.WriteHeader(http.StatusUnauthorized) return } From 5fa0703403355a81a224a1cfca3d36eb18e6f521 Mon Sep 17 00:00:00 2001 From: Anton Markelov Date: Thu, 26 Dec 2019 16:59:54 +1000 Subject: [PATCH 18/20] ADMIN-3023 add task slugs add proper error handling --- Gopkg.lock | 10 ++++- Gopkg.toml | 6 ++- api-docs.yml | 39 ++++++++++++++++ api/events.go | 4 +- api/login.go | 2 +- api/projects/environment.go | 2 +- api/projects/inventory.go | 2 +- api/projects/keys.go | 2 +- api/projects/project.go | 2 +- api/projects/projects.go | 2 +- api/projects/repository.go | 2 +- api/projects/templates.go | 45 ++++++++++++------- api/projects/users.go | 2 +- api/router.go | 5 ++- api/tasks/http.go | 20 ++++++++- api/user.go | 2 +- api/users.go | 2 +- db/Template.go | 3 +- db/migrations/v2.5.2.sql | 5 ++- deployment/docker/dev/Dockerfile | 5 ++- web/resources/pug/projects/templates/add.pug | 5 +++ web/resources/pug/projects/templates/list.pug | 2 + 22 files changed, 133 insertions(+), 36 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index bca322d13..ed8ceec94 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -27,9 +27,9 @@ [[projects]] branch = "master" - name = "github.com/castawaylabs/mulekick" + name = "github.com/strangeman/mulekick" packages = ["."] - revision = "7029fb389811e0f873c56cfbbda64d66af48b095" + revision = "3d90b180d25370ed05f3173b93757c53d75883b2" [[projects]] branch = "master" @@ -173,6 +173,12 @@ revision = "a6b93000bd219143c56c16e6cb1c4b91da3f224b" version = "v1.0" +[[projects]] + branch = "master" + name = "github.com/tjarratt/babble" + packages = ["."] + revision = "eecdf8c2339de49c56f0e3ac459077db829be7ee" + [[projects]] branch = "master" name = "github.com/mitchellh/mapstructure" diff --git a/Gopkg.toml b/Gopkg.toml index 0e4bbfff1..f5fe4aec5 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -4,7 +4,7 @@ [[constraint]] branch = "master" - name = "github.com/castawaylabs/mulekick" + name = "github.com/strangeman/mulekick" [[constraint]] name = "github.com/go-sql-driver/mysql" @@ -58,6 +58,10 @@ name = "gopkg.in/ldap.v2" version = "2.5.1" +[[constraint]] + name = "github.com/tjarratt/babble" + branch = "master" + [prune] go-tests = true unused-packages = true diff --git a/api-docs.yml b/api-docs.yml index dc18b6d8d..e6c8a7dd5 100644 --- a/api-docs.yml +++ b/api-docs.yml @@ -287,6 +287,8 @@ definitions: environment_id: type: integer minimum: 1 + slug: + type: string alias: type: string playbook: @@ -301,6 +303,8 @@ definitions: id: type: integer minimum: 1 + slug: + type: string ssh_key_id: type: integer minimum: 1 @@ -424,6 +428,13 @@ parameters: type: integer required: true x-example: 8 + slug: + name: slug + description: Slug + in: path + type: string + required: true + x-example: some-task paths: /ping: @@ -1213,6 +1224,34 @@ paths: description: Task queued schema: $ref: "#/definitions/Task" + /project/{project_id}/slug/{slug}: + parameters: + - $ref: "#/parameters/project_id" + - $ref: "#/parameters/slug" + post: + tags: + - project + summary: Starts a job via task slug + parameters: + - name: task + in: body + required: true + schema: + type: object + properties: + debug: + type: boolean + dry_run: + type: boolean + playbook: + type: string + environment: + type: string + responses: + 201: + description: Task queued + schema: + $ref: "#/definitions/Task" /project/{project_id}/tasks/last: parameters: - $ref: "#/parameters/project_id" diff --git a/api/events.go b/api/events.go index bc7d057b3..ef22f79a9 100644 --- a/api/events.go +++ b/api/events.go @@ -4,10 +4,10 @@ import ( "net/http" "github.com/ansible-semaphore/semaphore/db" - "github.com/castawaylabs/mulekick" + "github.com/ansible-semaphore/semaphore/util" "github.com/gorilla/context" "github.com/masterminds/squirrel" - "github.com/ansible-semaphore/semaphore/util" + "github.com/strangeman/mulekick" ) //nolint: gocyclo diff --git a/api/login.go b/api/login.go index 8e47510f8..9b007ae24 100644 --- a/api/login.go +++ b/api/login.go @@ -12,7 +12,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" + "github.com/strangeman/mulekick" sq "github.com/masterminds/squirrel" "golang.org/x/crypto/bcrypt" "gopkg.in/ldap.v2" diff --git a/api/projects/environment.go b/api/projects/environment.go index 9e4088eba..6c1947f37 100644 --- a/api/projects/environment.go +++ b/api/projects/environment.go @@ -7,9 +7,9 @@ import ( "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" "github.com/gorilla/context" "github.com/masterminds/squirrel" + "github.com/strangeman/mulekick" ) // EnvironmentMiddleware ensures an environment exists and loads it to the context diff --git a/api/projects/inventory.go b/api/projects/inventory.go index e65ffe7d0..bfbf42454 100644 --- a/api/projects/inventory.go +++ b/api/projects/inventory.go @@ -6,7 +6,7 @@ import ( "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" + "github.com/strangeman/mulekick" "github.com/gorilla/context" "github.com/masterminds/squirrel" "path/filepath" diff --git a/api/projects/keys.go b/api/projects/keys.go index 40eaeb97f..a32116b8c 100644 --- a/api/projects/keys.go +++ b/api/projects/keys.go @@ -6,7 +6,7 @@ import ( "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" + "github.com/strangeman/mulekick" "github.com/gorilla/context" "github.com/masterminds/squirrel" ) diff --git a/api/projects/project.go b/api/projects/project.go index 4ffaddf13..37bc15a0b 100644 --- a/api/projects/project.go +++ b/api/projects/project.go @@ -6,7 +6,7 @@ import ( "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" + "github.com/strangeman/mulekick" "github.com/gorilla/context" "github.com/masterminds/squirrel" ) diff --git a/api/projects/projects.go b/api/projects/projects.go index f0d868ac2..daabf1396 100644 --- a/api/projects/projects.go +++ b/api/projects/projects.go @@ -4,7 +4,7 @@ import ( "net/http" "github.com/ansible-semaphore/semaphore/db" - "github.com/castawaylabs/mulekick" + "github.com/strangeman/mulekick" "github.com/gorilla/context" "github.com/masterminds/squirrel" "time" diff --git a/api/projects/repository.go b/api/projects/repository.go index 3e8b1f3f4..81bce899e 100644 --- a/api/projects/repository.go +++ b/api/projects/repository.go @@ -8,7 +8,7 @@ import ( "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" + "github.com/strangeman/mulekick" "github.com/gorilla/context" "github.com/masterminds/squirrel" ) diff --git a/api/projects/templates.go b/api/projects/templates.go index ec82c7f04..33fa718ee 100644 --- a/api/projects/templates.go +++ b/api/projects/templates.go @@ -4,12 +4,14 @@ import ( "database/sql" "net/http" "strconv" + "strings" "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" "github.com/gorilla/context" "github.com/masterminds/squirrel" + "github.com/strangeman/mulekick" + "github.com/tjarratt/babble" ) // TemplatesMiddleware ensures a template exists and loads it to the context @@ -46,21 +48,22 @@ func GetTemplates(w http.ResponseWriter, r *http.Request) { } q := squirrel.Select("pt.id", - "pt.ssh_key_id", - "pt.project_id", - "pt.inventory_id", - "pt.repository_id", - "pt.environment_id", - "pt.alias", - "pt.playbook", - "pt.arguments", - "pt.override_args"). - From("project__template pt") + "pt.slug", + "pt.ssh_key_id", + "pt.project_id", + "pt.inventory_id", + "pt.repository_id", + "pt.environment_id", + "pt.alias", + "pt.playbook", + "pt.arguments", + "pt.override_args"). + From("project__template pt") switch sort { case "alias", "playbook": q = q.Where("pt.project_id=?", project.ID). - OrderBy("pt."+ sort + " " + order) + OrderBy("pt." + sort + " " + order) case "ssh_key": q = q.LeftJoin("access_key ak ON (pt.ssh_key_id = ak.id)"). Where("pt.project_id=?", project.ID). @@ -79,7 +82,7 @@ func GetTemplates(w http.ResponseWriter, r *http.Request) { OrderBy("pr.name " + order) default: q = q.Where("pt.project_id=?", project.ID). - OrderBy("pt.alias " + order) + OrderBy("pt.alias " + order) } query, args, err := q.ToSql() @@ -101,8 +104,16 @@ func AddTemplate(w http.ResponseWriter, r *http.Request) { return } - res, err := db.Mysql.Exec("insert into project__template set ssh_key_id=?, project_id=?, inventory_id=?, repository_id=?, environment_id=?, alias=?, playbook=?, arguments=?, override_args=?", template.SSHKeyID, project.ID, template.InventoryID, template.RepositoryID, template.EnvironmentID, template.Alias, template.Playbook, template.Arguments, template.OverrideArguments) + if template.Slug == "" { + template.Slug = babble.NewBabbler().Babble() + } + + res, err := db.Mysql.Exec("insert into project__template set slug=?, ssh_key_id=?, project_id=?, inventory_id=?, repository_id=?, environment_id=?, alias=?, playbook=?, arguments=?, override_args=?", template.Slug, template.SSHKeyID, project.ID, template.InventoryID, template.RepositoryID, template.EnvironmentID, template.Alias, template.Playbook, template.Arguments, template.OverrideArguments) if err != nil { + if strings.Contains(err.Error(), "Error 1062") { + mulekick.WriteJSON(w, http.StatusBadRequest, "{'error': 'slug must be unique'}") + return + } panic(err) } @@ -140,7 +151,11 @@ func UpdateTemplate(w http.ResponseWriter, r *http.Request) { template.Arguments = nil } - if _, err := db.Mysql.Exec("update project__template set ssh_key_id=?, inventory_id=?, repository_id=?, environment_id=?, alias=?, playbook=?, arguments=?, override_args=? where id=?", template.SSHKeyID, template.InventoryID, template.RepositoryID, template.EnvironmentID, template.Alias, template.Playbook, template.Arguments, template.OverrideArguments, oldTemplate.ID); err != nil { + if template.Slug == "" { + template.Slug = babble.NewBabbler().Babble() + } + + if _, err := db.Mysql.Exec("update project__template set slug=?, ssh_key_id=?, inventory_id=?, repository_id=?, environment_id=?, alias=?, playbook=?, arguments=?, override_args=? where id=?", template.Slug, template.SSHKeyID, template.InventoryID, template.RepositoryID, template.EnvironmentID, template.Alias, template.Playbook, template.Arguments, template.OverrideArguments, oldTemplate.ID); err != nil { panic(err) } diff --git a/api/projects/users.go b/api/projects/users.go index 7c5275f84..823149e7b 100644 --- a/api/projects/users.go +++ b/api/projects/users.go @@ -7,7 +7,7 @@ import ( "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" + "github.com/strangeman/mulekick" "github.com/gorilla/context" "github.com/masterminds/squirrel" ) diff --git a/api/router.go b/api/router.go index b89984373..71645f41f 100644 --- a/api/router.go +++ b/api/router.go @@ -8,10 +8,10 @@ import ( "github.com/ansible-semaphore/semaphore/api/sockets" "github.com/ansible-semaphore/semaphore/api/tasks" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" "github.com/gobuffalo/packr" "github.com/gorilla/mux" "github.com/russross/blackfriday" + "github.com/strangeman/mulekick" ) var publicAssets = packr.NewBox("../web/public") @@ -25,6 +25,7 @@ func JSONMiddleware(w http.ResponseWriter, r *http.Request) { func PlainTextMiddleware(w http.ResponseWriter, r *http.Request) { w.Header().Set("content-type", "text/plain; charset=utf-8") } + // Route declares all routes func Route() mulekick.Router { r := mulekick.New(mux.NewRouter(), mulekick.CorsMiddleware, JSONMiddleware) @@ -110,6 +111,8 @@ func Route() mulekick.Router { api.Put("/templates/{template_id}", projects.TemplatesMiddleware, projects.UpdateTemplate) api.Delete("/templates/{template_id}", projects.TemplatesMiddleware, projects.RemoveTemplate) + api.Post("/slug/{slug}", tasks.AddTask) + api.Get("/tasks", tasks.GetAllTasks) api.Get("/tasks/last", tasks.GetLastTasks) api.Post("/tasks", tasks.AddTask) diff --git a/api/tasks/http.go b/api/tasks/http.go index 51cab0129..03b69e58f 100644 --- a/api/tasks/http.go +++ b/api/tasks/http.go @@ -5,12 +5,15 @@ import ( "strconv" "time" + "database/sql" + log "github.com/Sirupsen/logrus" "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" "github.com/gorilla/context" + "github.com/gorilla/mux" "github.com/masterminds/squirrel" + "github.com/strangeman/mulekick" ) // AddTask inserts a task into the database and returns a header or returns error @@ -18,6 +21,8 @@ func AddTask(w http.ResponseWriter, r *http.Request) { project := context.Get(r, "project").(db.Project) user := context.Get(r, "user").(*db.User) + slug := mux.Vars(r)["slug"] + var taskObj db.Task if err := mulekick.Bind(w, r, &taskObj); err != nil { return @@ -27,6 +32,19 @@ func AddTask(w http.ResponseWriter, r *http.Request) { taskObj.Status = "waiting" taskObj.UserID = &user.ID + if slug != "" { + var template db.Template + if err := db.Mysql.SelectOne(&template, "select * from project__template where project_id=? and slug=?", project.ID, slug); err != nil { + if err == sql.ErrNoRows { + mulekick.WriteJSON(w, http.StatusNotFound, nil) + return + } + } else { + println(template.ID) + taskObj.TemplateID = template.ID + } + } + if err := db.Mysql.Insert(&taskObj); err != nil { util.LogErrorWithFields(err, log.Fields{"error": "Bad request. Cannot create new task"}) w.WriteHeader(http.StatusBadRequest) diff --git a/api/user.go b/api/user.go index a2bad4c11..daace3ba3 100644 --- a/api/user.go +++ b/api/user.go @@ -9,7 +9,7 @@ import ( "time" "github.com/ansible-semaphore/semaphore/db" - "github.com/castawaylabs/mulekick" + "github.com/strangeman/mulekick" "github.com/gorilla/context" "github.com/gorilla/mux" ) diff --git a/api/users.go b/api/users.go index da4a62861..01ae7c3d8 100644 --- a/api/users.go +++ b/api/users.go @@ -8,7 +8,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/ansible-semaphore/semaphore/db" "github.com/ansible-semaphore/semaphore/util" - "github.com/castawaylabs/mulekick" + "github.com/strangeman/mulekick" "github.com/gorilla/context" "golang.org/x/crypto/bcrypt" ) diff --git a/db/Template.go b/db/Template.go index 613e0ee49..35feb9167 100644 --- a/db/Template.go +++ b/db/Template.go @@ -2,7 +2,8 @@ package db // Template is a user defined model that is used to run a task type Template struct { - ID int `db:"id" json:"id"` + ID int `db:"id" json:"id"` + Slug string `db:"slug" json:"slug"` SSHKeyID int `db:"ssh_key_id" json:"ssh_key_id"` ProjectID int `db:"project_id" json:"project_id"` diff --git a/db/migrations/v2.5.2.sql b/db/migrations/v2.5.2.sql index ac61adc89..616e3740f 100644 --- a/db/migrations/v2.5.2.sql +++ b/db/migrations/v2.5.2.sql @@ -1 +1,4 @@ -alter table task add `arguments` text null; \ No newline at end of file +ALTER TABLE task add `arguments` text null; +ALTER TABLE `project__template` ADD slug varchar(64) not null default 'someslug' after id ; +UPDATE `project__template` set slug = CONCAT('slug-', CAST(id AS CHAR)); +ALTER TABLE `project__template` ADD UNIQUE (slug); diff --git a/deployment/docker/dev/Dockerfile b/deployment/docker/dev/Dockerfile index 3bc823346..92d33bd29 100644 --- a/deployment/docker/dev/Dockerfile +++ b/deployment/docker/dev/Dockerfile @@ -22,14 +22,15 @@ RUN apk add --no-cache git mysql-client python py-pip py-openssl openssl ca-cert chown -R semaphore:0 /etc/semaphore && \ ssh-keygen -t rsa -q -f "/root/.ssh/id_rsa" -N "" && \ ssh-keyscan -H github.com > /root/.ssh/known_hosts && \ - go get -u -v github.com/go-task/task/cmd/task + curl -sL https://taskfile.dev/install.sh | sh && \ + mv ./bin/task /usr/bin/task # Copy in app source WORKDIR ${APP_ROOT} COPY . ${APP_ROOT} RUN deployment/docker/dev/bin/install -USER 1000 +USER root EXPOSE 3000 CMD ["task", "watch"] \ No newline at end of file diff --git a/web/resources/pug/projects/templates/add.pug b/web/resources/pug/projects/templates/add.pug index 29f29457d..501505822 100644 --- a/web/resources/pug/projects/templates/add.pug +++ b/web/resources/pug/projects/templates/add.pug @@ -14,6 +14,11 @@ .col-sm-6 input.form-control(type="text" readonly="readonly" ng-model="tpl.id") + .form-group + label.control-label.col-sm-4 Template Slug + .col-sm-6 + input.form-control(type="text" placeholder="some-task" ng-model="tpl.slug") + .form-group label.control-label.col-sm-4 Playbook Name .col-sm-6 diff --git a/web/resources/pug/projects/templates/list.pug b/web/resources/pug/projects/templates/list.pug index cbef65bef..bc1496303 100644 --- a/web/resources/pug/projects/templates/list.pug +++ b/web/resources/pug/projects/templates/list.pug @@ -6,6 +6,7 @@ h3 Task Templates table.table.table-hover thead: tr th Alias + th Slug th Playbook th SSH Key th Inventory @@ -14,6 +15,7 @@ table.table.table-hover th   tbody: tr(ng-repeat="tpl in templates" ng-click="update(tpl)" style="cursor: pointer;" ng-if="!tpl.hidden || allShown") td {{ tpl.alias }} + td {{ tpl.slug }} td {{ tpl.playbook }} td {{ sshKeysAssoc[tpl.ssh_key_id].name }} td {{ inventoryAssoc[tpl.inventory_id].name }} From 93534b259de235d5e8fa4f479d20fbe406ceb52a Mon Sep 17 00:00:00 2001 From: Anton Markelov Date: Tue, 14 Jan 2020 12:56:24 +1000 Subject: [PATCH 19/20] tune taskfile --- Taskfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Taskfile.yml b/Taskfile.yml index 3d5eb8f63..507a0aaad 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -100,7 +100,7 @@ tasks: sh: git rev-parse --abbrev-ref HEAD DIRTY: # We must exclude the package-lock file as npm install can change it! - sh: git diff --exit-code --stat -- . ':(exclude)web/package-lock.json' ':(exclude)web/package.json' + sh: git diff --exit-code --stat -- . ':(exclude)web/package-lock.json' ':(exclude)web/package.json' ':(exclude)deployment/*' ':(exclude)Taskfile*' SHA: sh: git log --pretty=format:'%h' -n 1 TIMESTAMP: From 127fc036ec64a0c61fd26c8e70421d9e25eb4621 Mon Sep 17 00:00:00 2001 From: Anton Markelov Date: Tue, 28 Apr 2020 11:53:04 +1000 Subject: [PATCH 20/20] allow custom wrappers for ansible-galaxy and ansible-playbook --- api/tasks/runner.go | 6 +++--- util/config.go | 12 ++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/api/tasks/runner.go b/api/tasks/runner.go index cdfff364d..3c2f3e430 100644 --- a/api/tasks/runner.go +++ b/api/tasks/runner.go @@ -338,7 +338,7 @@ func (t *task) runGalaxy() error { "--force", } - cmd := exec.Command("ansible-galaxy", args...) //nolint: gas + cmd := exec.Command(util.Config.AnsibleGalaxyCommand, args...) //nolint: gas cmd.Dir = util.Config.TmpPath + "/repository_" + strconv.Itoa(t.repository.ID) gitSSHCommand := "ssh -o StrictHostKeyChecking=no -i " + t.repository.SSHKey.GetPath() @@ -364,7 +364,7 @@ func (t *task) listPlaybookHosts() (string, error) { } args = append(args, "--list-hosts") - cmd := exec.Command("ansible-playbook", args...) //nolint: gas + cmd := exec.Command(util.Config.AnsiblePlaybookCommand, args...) //nolint: gas cmd.Dir = util.Config.TmpPath + "/repository_" + strconv.Itoa(t.repository.ID) cmd.Env = t.envVars(util.Config.TmpPath, cmd.Dir, nil) @@ -388,7 +388,7 @@ func (t *task) runPlaybook() error { if err != nil { return err } - cmd := exec.Command("ansible-playbook", args...) //nolint: gas + cmd := exec.Command(util.Config.AnsiblePlaybookCommand, args...) //nolint: gas cmd.Dir = util.Config.TmpPath + "/repository_" + strconv.Itoa(t.repository.ID) cmd.Env = t.envVars(util.Config.TmpPath, cmd.Dir, nil) diff --git a/util/config.go b/util/config.go index d3fa5174a..7af369813 100644 --- a/util/config.go +++ b/util/config.go @@ -59,6 +59,10 @@ type configType struct { // semaphore stores ephemeral projects here TmpPath string `json:"tmp_path"` + // useful for overriding default ansible binaries + AnsibleGalaxyCommand string `json:"ansible_galaxy_command"` + AnsiblePlaybookCommand string `json:"ansible_playbook_command"` + // cookie hashing & encryption CookieHash string `json:"cookie_hash"` CookieEncryption string `json:"cookie_encryption"` @@ -205,6 +209,14 @@ func validateConfig() { Config.TmpPath = "/tmp/semaphore" } + if len(Config.AnsibleGalaxyCommand) == 0 { + Config.AnsibleGalaxyCommand = "ansible-galaxy" + } + + if len(Config.AnsiblePlaybookCommand) == 0 { + Config.AnsiblePlaybookCommand = "ansible-playbook" + } + if Config.MaxParallelTasks < 1 { Config.MaxParallelTasks = 10 }