diff --git a/.gitignore b/.gitignore index 08c76a1..3ba22a2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,10 @@ .vscode .task +.pytest* +site +__localbuild__ +_out* +_out/* +*.pyc +__pycache__ +.cache diff --git a/README.md b/README.md index fe07081..1654385 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,21 @@ Jenlib is Shared -Basic Usage: see [README.md](./var/README.md) +Basic Usage: see docs -## Job DSL +- [index.md](./docs/index.md) -- https://github.com/Ticketfly/jenkins-docker-examples -- https://www.praqma.com/stories/dockerized-jenkins-jobdsl/ -- https://github.com/thomasleveil/docker-jenkins-dsl-ready +## Parallel Execution -## References +- +- -- [Jenkins Shared Libraries](https://www.jenkins.io/doc/book/pipeline/shared-libraries/) \ No newline at end of file +## latest version + +- + +- +- + +- +- https://min.io/resources/docs/Spark-S3Select.pdf \ No newline at end of file diff --git a/Taskfile.yml b/Taskfile.yml index 30664c0..c0a0db0 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1,6 +1,16 @@ # https://taskfile.dev -version: '2' +version: '3' + +includes: + mkdocs: _infra/mkdocs_pack/mkdocs.tasks.yml + jenconda: + taskfile: dockers/jenkins-master/Taskfile.yml + dir: dockers/jenkins-master + e2e: ./tasks/e2e.tasks.yml + # docker: _infra/jenlib_lib/tasks/docker.tasks.yml + # tests: _infra/jenlib_lib/tasks/tests.tasks.yml + # ci: _infra/jenlib_lib/tasks/ci.tasks.yml vars: GREETING: Jenlib repo actions @@ -12,8 +22,76 @@ tasks: - task -l silent: true + info: + desc: Information about this dir + cmds: + - | + echo ' + # Jenlib taskfile + + ## Layout + + - `src` - jenkins shared lib 'jenlib' + - `jenpy` (jenlib) - python cli for jenkins, jenlib, gitlab, s3/http access + + ## Command Groups + + This repo is multi product repo + Each product have it's own taskfile + Taskfiles are copied to tasks dir from different ci-libs + Each taskfile express one group + + > Available Groups + + - jenlib - jenkins shared lib tasks + - docker - jenconda docker build + - mkdocs - documentation + - deps - environment tasks + + ## Group Commands + + Most Groups are + + + build: + desc: _ + cmds: + - echo "There is No build, but lot of tests" + readme: desc: _ cmds: - cat README.md | less + build:prepare: + desc: _ + cmds: + - rm -rf __localbuild__/jenbuild + - mkdir -p __localbuild__/jenbuild/jenlib + - rclone copyto vars __localbuild__/jenbuild/jenlib/vars + - rclone copyto version/version_prefix.txt __localbuild__/jenbuild/jenlib/version.txt + + build:do: + desc: _ + dir: __localbuild__/jenbuild/jenlib + cmds: + - zip -r ../jenlib.zip . + + publish:build: + desc: _ + cmds: + - rclone copyto __localbuild__/jenbuild/jenlib.zip mts3:moneytime-http/buildx/jenlib/jenlib.zip + + build:full: + desc: _ + cmds: + - task: build:prepare + - task: build:do + + full:cycle: + decs: _ + cmds: + - task: build:full + - task: publish:build + status: + - true # prevent accidental run diff --git a/_cicd/jenlib.build.groovy b/_cicd/jenlib.build.groovy new file mode 100644 index 0000000..3fe4418 --- /dev/null +++ b/_cicd/jenlib.build.groovy @@ -0,0 +1,16 @@ +@Library("jenlib@master") _ + +def kws = [:] + +LoadConfig( + kws, + dest: 'jenlib_config' + config_path: "version/jenlib_config.yml", + config_entry: "jenbuild_flow" +) + +JenFlowTopLevel( + kws, + taskfile: kws.jenlib_config.jenflow, + taskname: kws.jenlib_config.taskname +) diff --git a/_infra/mkdocs_pack/mkdocs.tasks.yml b/_infra/mkdocs_pack/mkdocs.tasks.yml new file mode 100644 index 0000000..540abd0 --- /dev/null +++ b/_infra/mkdocs_pack/mkdocs.tasks.yml @@ -0,0 +1,35 @@ +# https://taskfile.dev +version: '3' + +vars: + DIMAGE: "squidfunk/mkdocs-material" + DSERVE_PORT: 8690:8690 + DARGS: "-v ${PWD}:/docs" + +tasks: + _denv: &_ref__denv + desc: _ + # vars: + # TASK_LAST: "{{.TASK | splitList | last}}" + cmds: + - mkdir -p __localbuild__/site && chmod 777 __localbuild__/site + - docker run --rm {{ .DARGS }} {{.DMORE}} {{ .DIMAGE }} {{.DCMD}} + + build: + <<: *_ref__denv + vars: + DCMD: build + + serve: + <<: *_ref__denv + vars: + DCMD: serve + DMORE: -p {{.DSERVE_PORT}} + + shell: + <<: *_ref__denv + vars: + DMORE: + | + -p {{.DSERVE_PORT}} \ + -it --entrypoint /bin/sh diff --git a/_repos/.gitignore b/_repos/.gitignore new file mode 100644 index 0000000..1b653bd --- /dev/null +++ b/_repos/.gitignore @@ -0,0 +1,4 @@ +* +*.* +.* +!.gitignore diff --git a/e2e/data4test/_seed_job.pipe.groovy b/data4test/_seed_job.pipe.groovy similarity index 100% rename from e2e/data4test/_seed_job.pipe.groovy rename to data4test/_seed_job.pipe.groovy diff --git a/data4test/gitlab_seeds/batches/a_simple_batch/Taskfile.yml b/data4test/gitlab_seeds/batches/a_simple_batch/Taskfile.yml new file mode 100644 index 0000000..3845856 --- /dev/null +++ b/data4test/gitlab_seeds/batches/a_simple_batch/Taskfile.yml @@ -0,0 +1,44 @@ +# https://taskfile.dev + +version: '3' + +vars: + GREETING: Hello, World! + LBUILD_DIR: __localbuild__/sambuild + LART_NAME: art.tar + +tasks: + default: + cmds: + - echo "{{.GREETING}}" + silent: true + + prepare: + desc: prepare for build + cmds: + - rm -rf {{.LBUILD_DIR}} + - mkdir -p {{.LBUILD_DIR}}/bdir + - cp *.yml *.groovy {{.LBUILD_DIR}}/bdir + + build: + desc: do the build + dir: "{{.LBUILD_DIR}}/bdir" + cmds: + # - zip -r ../art.zip . + - tar -cvf ../{{.LART_NAME}} . + + test: + desc: test artifact + cmds: + - tar -tvf {{.LBUILD_DIR}}/{{.LART_NAME}} + # - unzip -vl {{.LBUILD_DIR}}/art.zip + + + ci-flow: + desc: full build cycle + jenlib: + ci-flow: body + cmds: + - task prepare + - task build + - task test diff --git a/data4test/gitlab_seeds/batches/a_simple_batch/pipe_500_cmds_minimal.groovy b/data4test/gitlab_seeds/batches/a_simple_batch/pipe_500_cmds_minimal.groovy new file mode 100644 index 0000000..38ea150 --- /dev/null +++ b/data4test/gitlab_seeds/batches/a_simple_batch/pipe_500_cmds_minimal.groovy @@ -0,0 +1,13 @@ +node { + stage('start'){ + sh 'echo START' + } + + stage('body'){ + sh 'find -name "*.groovy"' + } + + stage('finis'){ + sh 'echo FINSH' + } +} diff --git a/data4test/gitlab_seeds/batches/a_simple_batch/pipe_501_cmds_explicit.groovy b/data4test/gitlab_seeds/batches/a_simple_batch/pipe_501_cmds_explicit.groovy new file mode 100644 index 0000000..6b74abf --- /dev/null +++ b/data4test/gitlab_seeds/batches/a_simple_batch/pipe_501_cmds_explicit.groovy @@ -0,0 +1,33 @@ +def kwj = [:] + +def LBUILD_DIR='__localbuild__/sambuild' +def LART_NAME='art.tar' + +node { + stage('start'){ + sh 'echo START' + kwj.scmvars = checkout scm + } + + dir('data4test/gitlab_seeds/batches/a_simple_batch'){ + stage('task prepare'){ + sh "rm -rf ${LBUILD_DIR}" + sh "mkdir -p ${LBUILD_DIR}/bdir" + sh "cp *.yml *.groovy ${LBUILD_DIR}/bdir" + } + + stage('task build'){ + dir("${LBUILD_DIR}/bdir"){ + sh "tar -cvf ../${LART_NAME} ." + } + } + + stage('task test'){ + sh "tar -tvf ${LBUILD_DIR}/${LART_NAME}" + } + } + + stage('finis'){ + sh 'echo FINSH' + } +} diff --git a/data4test/gitlab_seeds/batches/a_simple_batch/pipe_502_tasks_explicit.groovy b/data4test/gitlab_seeds/batches/a_simple_batch/pipe_502_tasks_explicit.groovy new file mode 100644 index 0000000..e3a2304 --- /dev/null +++ b/data4test/gitlab_seeds/batches/a_simple_batch/pipe_502_tasks_explicit.groovy @@ -0,0 +1,24 @@ +def kwj = [:] + +node { + stage('start'){ + sh 'echo START' + kwj.scmvars = checkout scm + } + dir('data4test/gitlab_seeds/batches/a_simple_batch'){ + stage('task prepare'){ + sh 'task prepare' + } + + stage('task build'){ + sh 'task build' + } + + stage('task test'){ + sh 'task test' + } + } + stage('finis'){ + sh 'echo FINSH' + } +} diff --git a/data4test/gitlab_seeds/batches/a_simple_batch/pipe_503_tasks_from_yml.groovy b/data4test/gitlab_seeds/batches/a_simple_batch/pipe_503_tasks_from_yml.groovy new file mode 100644 index 0000000..21402d7 --- /dev/null +++ b/data4test/gitlab_seeds/batches/a_simple_batch/pipe_503_tasks_from_yml.groovy @@ -0,0 +1,27 @@ + +library identifier: 'jenlib@0.7.5', retriever: http( +credentialsId: 'asd', +httpURL: 'http://moneytime.yairdar.com/buildx/jenlib/jenlib.zip') + +def kwj = [ + 'scmvars': null, + 'task_parse_result': [:], +] + +node { + + stage('start'){ + sh 'echo START' + kwj.scmvars = checkout scm + } + + dir('data4test/gitlab_seeds/batches/a_simple_batch'){ + jen.step_stages_from_tasks( + kwj, '.' ,'Taskfile.yml', 'ci-flow' + ) + } + + stage('finis'){ + sh 'echo FINSH' + } +} diff --git a/data4test/gitlab_seeds/batches/c_complex_batch/Taskfile.yml b/data4test/gitlab_seeds/batches/c_complex_batch/Taskfile.yml new file mode 100644 index 0000000..6ae3f53 --- /dev/null +++ b/data4test/gitlab_seeds/batches/c_complex_batch/Taskfile.yml @@ -0,0 +1,71 @@ +version: "3" +output: prefixed + +tasks: + + build_with_config: &_ref_build_with_config + desc: _ + cmds: + - echo build with A={{.A}} B={{.B}} of {{.TASK}} + + build:cfg1: + <<: *_ref_build_with_config + vars: + A: 1 + B: 2 + + build:cfg2: + <<: *_ref_build_with_config + vars: + A: 0 + B: 555 + + build-p: + desc: _ + deps: + - build:cfg1 + - build:cfg2 + cmds: + - echo aggregate + + + test_with_config: &_ref_test_with_config + desc: _ + cmds: + - echo test with A={{.A}} B={{.B}} of {{.TASK}} + + test:cfg1: + <<: *_ref_test_with_config + vars: + A: 0 + B: 555 + test:cfg2: + <<: *_ref_test_with_config + vars: + A: 3 + B: 777777 + test:cfg3: + <<: *_ref_test_with_config + vars: + A: 3434 + B: 767 + + test-p: + desc: _ + deps: + - test:cfg1 + - test:cfg2 + - test:cfg3 + cmds: + - echo aggregate + + start: echo start + finish: echo finish + + ci-flow: + desc: _ + cmds: + - task start + - task build-p + - task test-p + - task finish \ No newline at end of file diff --git a/data4test/gitlab_seeds/batches/c_complex_batch/pipe_701_complex_pipes.groovy b/data4test/gitlab_seeds/batches/c_complex_batch/pipe_701_complex_pipes.groovy new file mode 100644 index 0000000..1c1b14e --- /dev/null +++ b/data4test/gitlab_seeds/batches/c_complex_batch/pipe_701_complex_pipes.groovy @@ -0,0 +1,89 @@ + + +def kwj = [ + 'scmvars': null, +] + +node { + stage('BigStage1'){ + sh 'echo ok' + } + stage('BigStage2'){ + parallel ( + 'cfg1': { + stage('fok1'){ + sh 'echo fok1' + } + stage('fok2'){ + sh 'echo fok2' + } + }, + 'cfg2': { + stage('wewill'){ + echo 'NN' + } + stage('wedo'){ + echo 'NS' + parallel ( + 'wedo:honne': { + stage('wedo:hhhhh'){ + sh 'echo gsfok1' + } + stage('wedo:bbbbbb'){ + sh 'echo gsfok1' + } + }, + 'wedo:twiter':{ + stage('wedo:ggaaaggg'){ + sh 'echo skfok2' + } + } + ) + } + stage('wedone'){ + echo 'NN' + } + }, + 'cfg3': { + stage('h234234'){ + echo 'NN' + } + stage('s23423'){ + echo 'NS' + // parallel ( + // 'g': { + // sh 'echo gsfok1' + // }, + // 'k':{ + // sh 'echo skfok2' + // } + // ) + } + } + ) + } + stage('middle'){ + echo 'NN' + } + stage('more'){ + echo 'NS' + parallel ( + 'honne1': { + stage('dsdfh12'){ + sh 'echo gsfok1' + } + stage('dsfffj12'){ + sh 'echo gsfok1' + } + }, + 'twiter1':{ + stage('hjkhju12'){ + sh 'echo skfok2' + } + } + ) + } + stage('report'){ + echo 'NN' + } +} \ No newline at end of file diff --git a/data4test/gitlab_seeds/batches/c_complex_batch/pipe_703_tasks_from_yml.groovy b/data4test/gitlab_seeds/batches/c_complex_batch/pipe_703_tasks_from_yml.groovy new file mode 100644 index 0000000..9a28fc2 --- /dev/null +++ b/data4test/gitlab_seeds/batches/c_complex_batch/pipe_703_tasks_from_yml.groovy @@ -0,0 +1,27 @@ + +library identifier: 'jenlib@0.7.5', retriever: http( +credentialsId: 'asd', +httpURL: 'http://moneytime.yairdar.com/buildx/jenlib/jenlib.zip') + +def kwj = [ + 'scmvars': null, + 'task_parse_result': [:], +] + +node { + + stage('start'){ + sh 'echo START' + kwj.scmvars = checkout scm + } + + dir('data4test/gitlab_seeds/batches/c_complex_batch'){ + jen.step_stages_from_tasks( + kwj, '.' ,'Taskfile.yml', 'ci-flow' + ) + } + + stage('finis'){ + sh 'echo FINSH' + } +} diff --git a/data4test/gitlab_seeds/batches/c_complex_batch/pipe_730_groovy_dynamic.groovy b/data4test/gitlab_seeds/batches/c_complex_batch/pipe_730_groovy_dynamic.groovy new file mode 100644 index 0000000..c009b74 --- /dev/null +++ b/data4test/gitlab_seeds/batches/c_complex_batch/pipe_730_groovy_dynamic.groovy @@ -0,0 +1,24 @@ +def kwj = [:] + +def LBUILD_DIR='__localbuild__/sambuild' +def LART_NAME='art.tar' + +node { + stage('start'){ + sh 'echo START' + kwj.scmvars = checkout scm + } + + stage('finis'){ + sh 'echo FINSH' + def res = evaluate 'def test() { "eval EVALTEST" }; return test()' + echo res + } + evaluate """ + stage('ini'){ + sh 'echo INI' + def res = evaluate 'def test() { "eval INCEPTION" }; return test()' + echo res + } + """ +} diff --git a/data4test/gitlab_seeds/jobs-defs/gitlab_job_dsl.groovy b/data4test/gitlab_seeds/jobs-defs/gitlab_job_dsl.groovy new file mode 100644 index 0000000..e86ff57 --- /dev/null +++ b/data4test/gitlab_seeds/jobs-defs/gitlab_job_dsl.groovy @@ -0,0 +1,119 @@ +private boolean isSandbox() { + def locationConfig = jenkins.model.JenkinsLocationConfiguration.get() + if (locationConfig != null && locationConfig.getUrl() != null) { + locationConfig.getUrl().contains("staging") + } else { + System.getenv("ENVIRONMENT") == "sandbox" + } +} + +List gitlab = [] +if (isSandbox()) { + gitlab = [ + [name: 'tests', displayName: 'Jenkins Tests', group: 'jenkins/tests'], + ] +} else { + + gitlab = [ + [ group: 'jenlib_sample', + name: 'jenlib_sample_name', + displayName: 'jenlib_sample_dname', + folder: 'GitlabFolders'] + ] +} + +def folders = gitlab.collect { it.folder }.unique() - null + +folders.each { + folder(it) { + } +} + +gitlab.each { Map org -> + gitlabOrgs(org).call() +} + +Closure gitlabOrgs(Map args = [:]) { + def config = [ + displayName: args.name, + group: args.name, + suppressDefaultJenkinsfile: false, + ] << args + def name = config.folder ? "${config.folder}/${config.name}" : config.name + GString orgDescription = "
${config.displayName} group projects" + + return { + organizationFolder(name) { + organizations { + displayName(config.displayName) + description(orgDescription) + gitLabSCMNavigator { + projectOwner(config.group) + // credentialsId('gitlab_ssh_key') + serverName('gitlab-9313') + traits { + subGroupProjectDiscoveryTrait() // discover projects inside subgroups + gitLabBranchDiscovery { + strategyId(3) // discover all branches + } + originMergeRequestDiscoveryTrait { + strategyId(1) // discover MRs and merge them with target branch + } + gitLabTagDiscovery() // discover tags + gitLFSPullTrait() + // gitLocalBranchTrait { + // localBranch("**") + // } + } + } + } + + // "Traits" ("Behaviours" in the GUI) that are NOT "declarative-compatible" + // For some 'traits, we need to configure this stuff by hand until JobDSL handles it + // https://issues.jenkins.io/browse/JENKINS-45504 + configure { + def traits = it / navigators / 'io.jenkins.plugins.gitlabbranchsource.GitLabSCMNavigator' / traits + traits << 'io.jenkins.plugins.gitlabbranchsource.ForkMergeRequestDiscoveryTrait' { + strategyId 2 + trust(class: 'io.jenkins.plugins.gitlabbranchsource.ForkMergeRequestDiscoveryTrait$TrustPermission') + } + } + // configure { + // def traits = it / sources / data / 'jenkins.branch.BranchSource' / source / traits + // traits << 'jenkins.plugins.git.traits.LocalBranchTrait' { + // localBranch('**') + // } + // configure { + // def traits = it / navigators / 'org.jenkinsci.plugins.github__branch__source.GitHubSCMNavigator' / traits + // traits << 'org.jenkinsci.plugins.github_branch_source.BranchDiscoveryTrait' { + // strategyId 1 + // } + // traits << 'org.jenkinsci.plugins.github_branch_source.ForkPullRequestDiscoveryTrait' { + // strategyId 2 + // trust(class: 'org.jenkinsci.plugins.github_branch_source.ForkPullRequestDiscoveryTrait$TrustEveryone') + // } + // traits << 'org.jenkinsci.plugins.github__branch__source.OriginPullRequestDiscoveryTrait' { + // strategyId 2 + // } + // } + orphanedItemStrategy { + discardOldItems { + daysToKeep(7) + numToKeep(10) + } + } + if (!isSandbox()) { + triggers { + periodicFolderTrigger { + interval('1d') + } + } + } + projectFactories { + workflowMultiBranchProjectFactory { + scriptPath('build.pipe.groovy') + } + } + } + } +} diff --git a/data4test/gitlab_seeds/jobs-defs/sample_job_dsl.groovy b/data4test/gitlab_seeds/jobs-defs/sample_job_dsl.groovy new file mode 100644 index 0000000..530ced1 --- /dev/null +++ b/data4test/gitlab_seeds/jobs-defs/sample_job_dsl.groovy @@ -0,0 +1,55 @@ + +def kws = [ + seed_job_branch: 'nextrelease.v0.7.6', + root_dir: "samples", + seed_job_pipes: [ + 'data4test/gitlab_seeds/batches/a_simple_batch/pipe_500_cmds_minimal.groovy', + 'data4test/gitlab_seeds/batches/a_simple_batch/pipe_501_cmds_explicit.groovy', + 'data4test/gitlab_seeds/batches/a_simple_batch/pipe_502_tasks_explicit.groovy', + 'data4test/gitlab_seeds/batches/a_simple_batch/pipe_503_tasks_from_yml.groovy', + 'data4test/gitlab_seeds/batches/c_complex_batch/pipe_701_complex_pipes.groovy', + 'data4test/gitlab_seeds/batches/c_complex_batch/pipe_703_tasks_from_yml.groovy', + 'data4test/gitlab_seeds/batches/c_complex_batch/pipe_730_groovy_dynamic.groovy', + ], + repo_path: "file:///repo" +] + + +folder(kws.root_dir){ + +} + +for (seed_job_pipe in kws.seed_job_pipes) +{ + def arr = seed_job_pipe.split('/') + def jname = arr[-1] + def jfname = "${kws.root_dir}/${jname}" + pipelineJob(jfname) { + description() + keepDependencies(false) + definition { + cpsScm { + scm { + git { + remote { + url(kws.repo_path) + } + branch("*/${kws.seed_job_branch}") + } + } + scriptPath("${seed_job_pipe}") + } + } + // environmentVariables { + // envs(jobs_def_dir: "\${jobs_def_dir}") + // keepBuildVariables(true) + // } + disabled(false) + configure { + it / 'properties' / 'com.sonyericsson.rebuild.RebuildSettings' { + 'autoRebuild'('false') + 'rebuildDisabled'('false') + } + } + } +} \ No newline at end of file diff --git a/data4test/gitlab_seeds/jobs-registrator.pipe.groovy b/data4test/gitlab_seeds/jobs-registrator.pipe.groovy new file mode 100644 index 0000000..5d742c7 --- /dev/null +++ b/data4test/gitlab_seeds/jobs-registrator.pipe.groovy @@ -0,0 +1,31 @@ +// @Library(value="Jenkins_Jenlib@master",changelog=false) + +// ===== /SHARED PART ==== +// --- globas vars --- +// expected vars +def jobs_def_dir = env.jobs_def_dir ?: 'data4test/gitlab_seeds/jobs-defs' + +// kw.jnode_label +node() { timestamps { + + stage('fetch') { + deleteDir() + echo 'Fetch source' + def scmvars = checkout scm + } + + stage('create:jobs:local'){ + def jobs_def_pattern = "${jobs_def_dir}/*.groovy" + echo "@info: action.jobdsl=create:jobs:local target.file.pattern='${jobs_def_pattern}'" + jobDsl targets: [ + jobs_def_pattern + ].join('\n'), + lookupStrategy: 'SEED_JOB', + removedJobAction: 'DELETE', + removedViewAction: 'DELETE' + } + stage('register:arts') { + sh "find ${jobs_def_dir}" + archiveArtifacts "${jobs_def_dir}/*" + } +}} diff --git a/data4test/gitlab_tests/jdsl/gitlab-job-dsl.groovy b/data4test/gitlab_tests/jdsl/gitlab-job-dsl.groovy new file mode 100644 index 0000000..30f4ce8 --- /dev/null +++ b/data4test/gitlab_tests/jdsl/gitlab-job-dsl.groovy @@ -0,0 +1,96 @@ +private boolean isSandbox() { + def locationConfig = jenkins.model.JenkinsLocationConfiguration.get() + if (locationConfig != null && locationConfig.getUrl() != null) { + locationConfig.getUrl().contains("staging") + } else { + System.getenv("ENVIRONMENT") == "sandbox" + } +} + +List gitlab = [] +if (isSandbox()) { + gitlab = [ + [name: 'tests', displayName: 'Jenkins Tests', group: 'jenkins/tests'], + ] +} else { + + gitlab = [ + [group: 'jenlib_sample' name: 'jenlib_sample_name', displayName: 'jenlib_sample_dname' folder: 'GitlabFolders'], + ] +} + +def folders = gitlab.collect { it.folder }.unique() - null + +folders.each { + folder(it) { + } +} + +gitlab.each { Map org -> + gitlabOrgs(org).call() +} + +Closure gitlabOrgs(Map args = [:]) { + def config = [ + displayName: args.name, + group: args.name, + suppressDefaultJenkinsfile: false, + ] << args + def name = config.folder ? "${config.folder}/${config.name}" : config.name + GString orgDescription = "
${config.displayName} group projects" + + return { + organizationFolder(name) { + organizations { + displayName(config.displayName) + description(orgDescription) + gitLabSCMNavigator { + projectOwner(config.group) + // credentialsId('gitlab_ssh_key') + serverName('gitlab-9313') + traits { + subGroupProjectDiscoveryTrait() // discover projects inside subgroups + gitLabBranchDiscovery { + strategyId(3) // discover all branches + } + originMergeRequestDiscoveryTrait { + strategyId(1) // discover MRs and merge them with target branch + } + gitLabTagDiscovery() // discover tags + gitLFSPullTrait() + } + } + } + + // "Traits" ("Behaviours" in the GUI) that are NOT "declarative-compatible" + // For some 'traits, we need to configure this stuff by hand until JobDSL handles it + // https://issues.jenkins.io/browse/JENKINS-45504 + configure { + def traits = it / navigators / 'io.jenkins.plugins.gitlabbranchsource.GitLabSCMNavigator' / traits + traits << 'io.jenkins.plugins.gitlabbranchsource.ForkMergeRequestDiscoveryTrait' { + strategyId 2 + trust(class: 'io.jenkins.plugins.gitlabbranchsource.ForkMergeRequestDiscoveryTrait$TrustPermission') + } + } + + orphanedItemStrategy { + discardOldItems { + daysToKeep(7) + numToKeep(10) + } + } + if (!isSandbox()) { + triggers { + periodicFolderTrigger { + interval('1d') + } + } + } + projectFactories { + workflowMultiBranchProjectFactory { + scriptPath('Jenkinsfile') + } + } + } + } +} diff --git a/data4test/jenlib_pure/test_basic_task/Taskfile.jenflow.yml b/data4test/jenlib_pure/test_basic_task/Taskfile.jenflow.yml new file mode 100644 index 0000000..28c619c --- /dev/null +++ b/data4test/jenlib_pure/test_basic_task/Taskfile.jenflow.yml @@ -0,0 +1,27 @@ + +entries: + stages_from_task: + stages: + - stage: task prepare + steps: + - step: sh + mparams: {script: task prepare} + - stage: task ci-flow + steps: + - step: sh + mparams: {script: task ci-flow} + - stage: task report + steps: + - step: sh + mparams: {script: task report} + + stage_with_steps_from_task: + stages: + - stage: top-level + steps: + - step: sh + mparams: {script: task prepare} + - step: sh + mparams: {script: task ci-flow} + - step: sh + mparams: {script: task report} \ No newline at end of file diff --git a/data4test/jenlib_pure/test_basic_task/Taskfile.yml b/data4test/jenlib_pure/test_basic_task/Taskfile.yml new file mode 100644 index 0000000..0ad7197 --- /dev/null +++ b/data4test/jenlib_pure/test_basic_task/Taskfile.yml @@ -0,0 +1,31 @@ +# https://taskfile.dev + +version: '3' + +GREETING: Hello, World! + +tasks: + default: + cmds: + - echo "{{.GREETING}}" + silent: true + + prepare: + cmds: + - pwd + + report: + cmds: + - ls -alh + + ci-flow: + cmds: + - echo ci-flow init + - echo ci-flow over + + top-level: + desc: _ + cmds: + - task prepare + - task ci-flow + - task report diff --git a/data4test/new_parl/devsample_pipe.groovy b/data4test/new_parl/devsample_pipe.groovy new file mode 100644 index 0000000..1c1b14e --- /dev/null +++ b/data4test/new_parl/devsample_pipe.groovy @@ -0,0 +1,89 @@ + + +def kwj = [ + 'scmvars': null, +] + +node { + stage('BigStage1'){ + sh 'echo ok' + } + stage('BigStage2'){ + parallel ( + 'cfg1': { + stage('fok1'){ + sh 'echo fok1' + } + stage('fok2'){ + sh 'echo fok2' + } + }, + 'cfg2': { + stage('wewill'){ + echo 'NN' + } + stage('wedo'){ + echo 'NS' + parallel ( + 'wedo:honne': { + stage('wedo:hhhhh'){ + sh 'echo gsfok1' + } + stage('wedo:bbbbbb'){ + sh 'echo gsfok1' + } + }, + 'wedo:twiter':{ + stage('wedo:ggaaaggg'){ + sh 'echo skfok2' + } + } + ) + } + stage('wedone'){ + echo 'NN' + } + }, + 'cfg3': { + stage('h234234'){ + echo 'NN' + } + stage('s23423'){ + echo 'NS' + // parallel ( + // 'g': { + // sh 'echo gsfok1' + // }, + // 'k':{ + // sh 'echo skfok2' + // } + // ) + } + } + ) + } + stage('middle'){ + echo 'NN' + } + stage('more'){ + echo 'NS' + parallel ( + 'honne1': { + stage('dsdfh12'){ + sh 'echo gsfok1' + } + stage('dsfffj12'){ + sh 'echo gsfok1' + } + }, + 'twiter1':{ + stage('hjkhju12'){ + sh 'echo skfok2' + } + } + ) + } + stage('report'){ + echo 'NN' + } +} \ No newline at end of file diff --git a/e2e/data4test/parallel_tasks_jen/Taskfile.yml b/data4test/parallel_tasks_jen/Taskfile.yml similarity index 100% rename from e2e/data4test/parallel_tasks_jen/Taskfile.yml rename to data4test/parallel_tasks_jen/Taskfile.yml diff --git a/e2e/data4test/parallel_tasks_jen/build.groovy b/data4test/parallel_tasks_jen/build.groovy similarity index 92% rename from e2e/data4test/parallel_tasks_jen/build.groovy rename to data4test/parallel_tasks_jen/build.groovy index 3ff8a8f..09b2f1f 100644 --- a/e2e/data4test/parallel_tasks_jen/build.groovy +++ b/data4test/parallel_tasks_jen/build.groovy @@ -27,7 +27,7 @@ node() {timestamps { jen.desc_from_commits(currentBuild, jg) } - catchError { dir('e2e/data4test/parallel_tasks_jen'){ + catchError { dir('data4test/parallel_tasks_jen'){ jen.step_stages_from_tasks( jg, jg.prj_dir, 'Taskfile.yml', 'ci-flow') diff --git a/data4test/pipe_samples/Taskfile.yml b/data4test/pipe_samples/Taskfile.yml new file mode 100644 index 0000000..6ae3f53 --- /dev/null +++ b/data4test/pipe_samples/Taskfile.yml @@ -0,0 +1,71 @@ +version: "3" +output: prefixed + +tasks: + + build_with_config: &_ref_build_with_config + desc: _ + cmds: + - echo build with A={{.A}} B={{.B}} of {{.TASK}} + + build:cfg1: + <<: *_ref_build_with_config + vars: + A: 1 + B: 2 + + build:cfg2: + <<: *_ref_build_with_config + vars: + A: 0 + B: 555 + + build-p: + desc: _ + deps: + - build:cfg1 + - build:cfg2 + cmds: + - echo aggregate + + + test_with_config: &_ref_test_with_config + desc: _ + cmds: + - echo test with A={{.A}} B={{.B}} of {{.TASK}} + + test:cfg1: + <<: *_ref_test_with_config + vars: + A: 0 + B: 555 + test:cfg2: + <<: *_ref_test_with_config + vars: + A: 3 + B: 777777 + test:cfg3: + <<: *_ref_test_with_config + vars: + A: 3434 + B: 767 + + test-p: + desc: _ + deps: + - test:cfg1 + - test:cfg2 + - test:cfg3 + cmds: + - echo aggregate + + start: echo start + finish: echo finish + + ci-flow: + desc: _ + cmds: + - task start + - task build-p + - task test-p + - task finish \ No newline at end of file diff --git a/data4test/pipe_samples/meci.task.sample.yml b/data4test/pipe_samples/meci.task.sample.yml new file mode 100644 index 0000000..b5c6230 --- /dev/null +++ b/data4test/pipe_samples/meci.task.sample.yml @@ -0,0 +1,69 @@ +version: "3" +tasks: + + build_with_config: &_ref_build_with_config + desc: _ + cmds: + - echo build with A={{.A}} B={{.B}} of {{.TASK}} + + build:cfg1: + <<: *_ref_build_with_config + vars: + A: 1 + B: 2 + + build:cfg2: + <<: *_ref_build_with_config + vars: + A: 0 + B: 555 + + build-p: + desc: _ + deps: + - build:cfg1 + - build:cfg2 + cmds: + - echo aggregate + + + test_with_config: &_ref_test_with_config + desc: _ + cmds: + - echo test with A={{.A}} B={{.B}} of {{.TASK}} + + test:cfg1: + <<: *_ref_test_with_config + vars: + A: 0 + B: 555 + test:cfg2: + <<: *_ref_test_with_config + vars: + A: 3 + B: 777777 + test:cfg3: + <<: *_ref_test_with_config + vars: + A: 3434 + B: 767 + + test-p: + desc: _ + deps: + - test:cfg1 + - test:cfg2 + - test:cfg3 + cmds: + - echo aggregate + + start: echo start + finish: echo finish + + ci-flow: + desc: _ + cmds: + - task start + - task: build-p + - task: test-p + - task finish \ No newline at end of file diff --git a/data4test/pipe_samples/multi_cfg.groovy b/data4test/pipe_samples/multi_cfg.groovy new file mode 100644 index 0000000..821513c --- /dev/null +++ b/data4test/pipe_samples/multi_cfg.groovy @@ -0,0 +1,53 @@ +def kwj = [ + 'scmvars': null, +] + +node { + stage('BigStage1'){ + sh 'echo ok' + } + stage('BigStage2'){ + parallel ( + 'cfg1': { + stage('fok1'){ + sh 'echo fok1' + } + stage('fok2'){ + sh 'echo fok2' + } + }, + 'cfg2': { + stage('h'){ + echo 'NN' + } + stage('s'){ + echo 'NS' + // parallel ( + // 'g': { + // sh 'echo gsfok1' + // }, + // 'k':{ + // sh 'echo skfok2' + // } + // ) + } + }, + 'cfg3': { + stage('h234234'){ + echo 'NN' + } + stage('s23423'){ + echo 'NS' + // parallel ( + // 'g': { + // sh 'echo gsfok1' + // }, + // 'k':{ + // sh 'echo skfok2' + // } + // ) + } + } + ) + } +} \ No newline at end of file diff --git a/data4test/pipe_samples/package.task.sample copy.yml b/data4test/pipe_samples/package.task.sample copy.yml new file mode 100644 index 0000000..eb9850c --- /dev/null +++ b/data4test/pipe_samples/package.task.sample copy.yml @@ -0,0 +1,38 @@ +tasks: + + + test_with_sources + + build-p: + desc: _ + deps: + - task: build-with-config + vars: + A: 1 + - task: tests:run-coverage-p + cmds: + - echo aggregate + + test-p: + desc: _ + deps: + - task: build-with-config + vars: + A: 1 + - task: build-with-config + vars: + A: 2 + cmds: + - echo aggregate + + ci-wf: + desc: _ + cmds: + - task: jenkins:repo:ensure + - task: ensure:deps + - task: test:before:build + - task: build-p + - task: install-build-p + - task: test-p + - task: finilize + - task: report \ No newline at end of file diff --git a/data4test/pipe_samples/somemore.task.sample copy 2.yml b/data4test/pipe_samples/somemore.task.sample copy 2.yml new file mode 100644 index 0000000..c289495 --- /dev/null +++ b/data4test/pipe_samples/somemore.task.sample copy 2.yml @@ -0,0 +1,41 @@ +tasks: + + + test_with_sources + + build-with-config: + + + build-p: + desc: _ + deps: + - task: build-with-config + vars: + A: 1 + - task: tests:run-coverage-p + cmds: + - echo aggregate + + test-p: + desc: _ + deps: + - task: build-with-config + vars: + A: 1 + - task: build-with-config + vars: + A: 2 + cmds: + - echo aggregate + + ci-wf: + desc: _ + cmds: + - task: jenkins:repo:ensure + - task: ensure:deps + - task: test:before:build + - task: build-p + - task: install-build-p + - task: test-p + - task: finilize + - task: report \ No newline at end of file diff --git a/data4test/settings/agen.yml b/data4test/settings/agen.yml new file mode 100644 index 0000000..ddaf7f1 --- /dev/null +++ b/data4test/settings/agen.yml @@ -0,0 +1,13 @@ + +options: + native: + pipe_type: native + node_label: algo-ci-generic-deb8-x86_64 + k8s: + pipe_type: k8s + pod_template_yaml: _infra/mxpack_lib/cicd/package.deliver.pipe.pod.tmpl.yaml + cloud: k8s-objx-prod-28102020 + container: mxcc-prod + # FIX: change to objx + ssh-agent: yairda-ssh-gitlab +default: k8s diff --git a/data4test/zold_gitlab_seeds/devsample_pipe.groovy b/data4test/zold_gitlab_seeds/devsample_pipe.groovy new file mode 100644 index 0000000..101d70a --- /dev/null +++ b/data4test/zold_gitlab_seeds/devsample_pipe.groovy @@ -0,0 +1,49 @@ +@Library(value="Jenkins_Jenlib",changelog=false) + +// pipeline { +// agent any +// stages { +// stage('Source') { +// steps { +// sh 'ls' +// sh 'pwd' +// } +// } +// stage('MULTI') { +// steps { +// jen.step_stages_from_tasks( +// kwj, '.' ,'Taskfile.yml', 'ci-flow' +// ) +// } +// } +// } +// } + +def kwj = [ + 'scmvars': null, + 'task_parse_result': [:], +] + +node { + + stage('start'){ + sh 'echo START' + kwj.scmvars = checkout scm + sh 'task --version' + sh 'task -l' + + } + + dir('data4test/pipe_samples'){ + stage('test'){ + sh 'task start' + } + jen.step_stages_from_tasks( + kwj, '.' ,'Taskfile.yml', 'ci-flow' + ) + } + + stage('finis'){ + sh 'echo FINSH' + } +} diff --git a/data4test/zold_gitlab_seeds/devsample_pipe_decl.groovy b/data4test/zold_gitlab_seeds/devsample_pipe_decl.groovy new file mode 100644 index 0000000..85399fe --- /dev/null +++ b/data4test/zold_gitlab_seeds/devsample_pipe_decl.groovy @@ -0,0 +1,19 @@ +@Library(value="Jenkins_Jenlib@",changelog=false) _ + +pipeline { + agent any + stages { + stage('Example Build') { + steps { + echo 'Hello, Maven' + } + } + // stage('Example Test') { + // steps { + // jen.step_stages_from_tasks( + // kwj, '.' ,'Taskfile.yml', 'ci-flow' + // ) + // } + // } + } +} \ No newline at end of file diff --git a/data4test/zold_gitlab_seeds/devsample_pipe_scripted.groovy b/data4test/zold_gitlab_seeds/devsample_pipe_scripted.groovy new file mode 100644 index 0000000..4d65ec8 --- /dev/null +++ b/data4test/zold_gitlab_seeds/devsample_pipe_scripted.groovy @@ -0,0 +1,31 @@ +@Library(value="Jenkins_Jenlib",changelog=false) + + +def kwj = [ + 'scmvars': null, + 'task_parse_result': [:], +] + +node { + + stage('start'){ + sh 'echo START' + kwj.scmvars = checkout scm + sh 'task --version' + sh 'task -l' + + } + + dir('data4test/pipe_samples'){ + stage('test'){ + sh 'task start' + } + jen.step_stages_from_tasks( + kwj, '.' ,'Taskfile.yml', 'ci-flow' + ) + } + + stage('finis'){ + sh 'echo FINSH' + } +} \ No newline at end of file diff --git a/data4test/zold_gitlab_seeds/jobs-defs/gitlab_job_dsl.groovy b/data4test/zold_gitlab_seeds/jobs-defs/gitlab_job_dsl.groovy new file mode 100644 index 0000000..e86ff57 --- /dev/null +++ b/data4test/zold_gitlab_seeds/jobs-defs/gitlab_job_dsl.groovy @@ -0,0 +1,119 @@ +private boolean isSandbox() { + def locationConfig = jenkins.model.JenkinsLocationConfiguration.get() + if (locationConfig != null && locationConfig.getUrl() != null) { + locationConfig.getUrl().contains("staging") + } else { + System.getenv("ENVIRONMENT") == "sandbox" + } +} + +List gitlab = [] +if (isSandbox()) { + gitlab = [ + [name: 'tests', displayName: 'Jenkins Tests', group: 'jenkins/tests'], + ] +} else { + + gitlab = [ + [ group: 'jenlib_sample', + name: 'jenlib_sample_name', + displayName: 'jenlib_sample_dname', + folder: 'GitlabFolders'] + ] +} + +def folders = gitlab.collect { it.folder }.unique() - null + +folders.each { + folder(it) { + } +} + +gitlab.each { Map org -> + gitlabOrgs(org).call() +} + +Closure gitlabOrgs(Map args = [:]) { + def config = [ + displayName: args.name, + group: args.name, + suppressDefaultJenkinsfile: false, + ] << args + def name = config.folder ? "${config.folder}/${config.name}" : config.name + GString orgDescription = "
${config.displayName} group projects" + + return { + organizationFolder(name) { + organizations { + displayName(config.displayName) + description(orgDescription) + gitLabSCMNavigator { + projectOwner(config.group) + // credentialsId('gitlab_ssh_key') + serverName('gitlab-9313') + traits { + subGroupProjectDiscoveryTrait() // discover projects inside subgroups + gitLabBranchDiscovery { + strategyId(3) // discover all branches + } + originMergeRequestDiscoveryTrait { + strategyId(1) // discover MRs and merge them with target branch + } + gitLabTagDiscovery() // discover tags + gitLFSPullTrait() + // gitLocalBranchTrait { + // localBranch("**") + // } + } + } + } + + // "Traits" ("Behaviours" in the GUI) that are NOT "declarative-compatible" + // For some 'traits, we need to configure this stuff by hand until JobDSL handles it + // https://issues.jenkins.io/browse/JENKINS-45504 + configure { + def traits = it / navigators / 'io.jenkins.plugins.gitlabbranchsource.GitLabSCMNavigator' / traits + traits << 'io.jenkins.plugins.gitlabbranchsource.ForkMergeRequestDiscoveryTrait' { + strategyId 2 + trust(class: 'io.jenkins.plugins.gitlabbranchsource.ForkMergeRequestDiscoveryTrait$TrustPermission') + } + } + // configure { + // def traits = it / sources / data / 'jenkins.branch.BranchSource' / source / traits + // traits << 'jenkins.plugins.git.traits.LocalBranchTrait' { + // localBranch('**') + // } + // configure { + // def traits = it / navigators / 'org.jenkinsci.plugins.github__branch__source.GitHubSCMNavigator' / traits + // traits << 'org.jenkinsci.plugins.github_branch_source.BranchDiscoveryTrait' { + // strategyId 1 + // } + // traits << 'org.jenkinsci.plugins.github_branch_source.ForkPullRequestDiscoveryTrait' { + // strategyId 2 + // trust(class: 'org.jenkinsci.plugins.github_branch_source.ForkPullRequestDiscoveryTrait$TrustEveryone') + // } + // traits << 'org.jenkinsci.plugins.github__branch__source.OriginPullRequestDiscoveryTrait' { + // strategyId 2 + // } + // } + orphanedItemStrategy { + discardOldItems { + daysToKeep(7) + numToKeep(10) + } + } + if (!isSandbox()) { + triggers { + periodicFolderTrigger { + interval('1d') + } + } + } + projectFactories { + workflowMultiBranchProjectFactory { + scriptPath('build.pipe.groovy') + } + } + } + } +} diff --git a/data4test/zold_gitlab_seeds/jobs-defs/sample_job_dsl.groovy b/data4test/zold_gitlab_seeds/jobs-defs/sample_job_dsl.groovy new file mode 100644 index 0000000..a7cdb9c --- /dev/null +++ b/data4test/zold_gitlab_seeds/jobs-defs/sample_job_dsl.groovy @@ -0,0 +1,37 @@ + + + +def kws = [ + seed_job_branch: 'yairda.presentaion', + seed_job_pipe: 'data4test/gitlab_seeds/devsample_pipe.groovy', + repo_path: "file:///repo" +] + +pipelineJob("devsample") { + description() + keepDependencies(false) + definition { + cpsScm { + scm { + git { + remote { + url(kws.repo_path) + } + branch("*/${kws.seed_job_branch}") + } + } + scriptPath("${kws.seed_job_pipe}") + } + } + // environmentVariables { + // envs(jobs_def_dir: "\${jobs_def_dir}") + // keepBuildVariables(true) + // } + disabled(false) + configure { + it / 'properties' / 'com.sonyericsson.rebuild.RebuildSettings' { + 'autoRebuild'('false') + 'rebuildDisabled'('false') + } + } +} \ No newline at end of file diff --git a/data4test/zold_gitlab_seeds/jobs-registrator.pipe.groovy b/data4test/zold_gitlab_seeds/jobs-registrator.pipe.groovy new file mode 100644 index 0000000..5d742c7 --- /dev/null +++ b/data4test/zold_gitlab_seeds/jobs-registrator.pipe.groovy @@ -0,0 +1,31 @@ +// @Library(value="Jenkins_Jenlib@master",changelog=false) + +// ===== /SHARED PART ==== +// --- globas vars --- +// expected vars +def jobs_def_dir = env.jobs_def_dir ?: 'data4test/gitlab_seeds/jobs-defs' + +// kw.jnode_label +node() { timestamps { + + stage('fetch') { + deleteDir() + echo 'Fetch source' + def scmvars = checkout scm + } + + stage('create:jobs:local'){ + def jobs_def_pattern = "${jobs_def_dir}/*.groovy" + echo "@info: action.jobdsl=create:jobs:local target.file.pattern='${jobs_def_pattern}'" + jobDsl targets: [ + jobs_def_pattern + ].join('\n'), + lookupStrategy: 'SEED_JOB', + removedJobAction: 'DELETE', + removedViewAction: 'DELETE' + } + stage('register:arts') { + sh "find ${jobs_def_dir}" + archiveArtifacts "${jobs_def_dir}/*" + } +}} diff --git a/e2e/gzvulon__base_docker/.gitignore b/dockers/gzvulon__base_docker/.gitignore similarity index 100% rename from e2e/gzvulon__base_docker/.gitignore rename to dockers/gzvulon__base_docker/.gitignore diff --git a/e2e/gzvulon__base_docker/README.md b/dockers/gzvulon__base_docker/README.md similarity index 100% rename from e2e/gzvulon__base_docker/README.md rename to dockers/gzvulon__base_docker/README.md diff --git a/e2e/gzvulon__base_docker/Taskfile.yml b/dockers/gzvulon__base_docker/Taskfile.yml similarity index 100% rename from e2e/gzvulon__base_docker/Taskfile.yml rename to dockers/gzvulon__base_docker/Taskfile.yml diff --git a/e2e/gzvulon__base_docker/docker.parts/0210_shell_setup_bash.part.Dockerfile b/dockers/gzvulon__base_docker/docker.parts/0210_shell_setup_bash.part.Dockerfile similarity index 100% rename from e2e/gzvulon__base_docker/docker.parts/0210_shell_setup_bash.part.Dockerfile rename to dockers/gzvulon__base_docker/docker.parts/0210_shell_setup_bash.part.Dockerfile diff --git a/e2e/gzvulon__base_docker/docker.parts/0211_create_user.part.Dockerfile b/dockers/gzvulon__base_docker/docker.parts/0211_create_user.part.Dockerfile similarity index 100% rename from e2e/gzvulon__base_docker/docker.parts/0211_create_user.part.Dockerfile rename to dockers/gzvulon__base_docker/docker.parts/0211_create_user.part.Dockerfile diff --git a/e2e/gzvulon__base_docker/docker.parts/0230_docker-client.part.Dockerfile b/dockers/gzvulon__base_docker/docker.parts/0230_docker-client.part.Dockerfile similarity index 100% rename from e2e/gzvulon__base_docker/docker.parts/0230_docker-client.part.Dockerfile rename to dockers/gzvulon__base_docker/docker.parts/0230_docker-client.part.Dockerfile diff --git a/e2e/gzvulon__base_docker/docker.parts/0330_miniconda_new.part.Dockerfile b/dockers/gzvulon__base_docker/docker.parts/0330_miniconda_new.part.Dockerfile similarity index 100% rename from e2e/gzvulon__base_docker/docker.parts/0330_miniconda_new.part.Dockerfile rename to dockers/gzvulon__base_docker/docker.parts/0330_miniconda_new.part.Dockerfile diff --git a/e2e/gzvulon__base_docker/docker.parts/0340_dtask.part.Dockerfile b/dockers/gzvulon__base_docker/docker.parts/0340_dtask.part.Dockerfile similarity index 100% rename from e2e/gzvulon__base_docker/docker.parts/0340_dtask.part.Dockerfile rename to dockers/gzvulon__base_docker/docker.parts/0340_dtask.part.Dockerfile diff --git a/e2e/gzvulon__base_docker/docker.parts/0350_python_23_conda.part.Dockerfile b/dockers/gzvulon__base_docker/docker.parts/0350_python_23_conda.part.Dockerfile similarity index 100% rename from e2e/gzvulon__base_docker/docker.parts/0350_python_23_conda.part.Dockerfile rename to dockers/gzvulon__base_docker/docker.parts/0350_python_23_conda.part.Dockerfile diff --git a/e2e/gzvulon__base_docker/docker.parts/0445_autobuild.part.Dockerfile b/dockers/gzvulon__base_docker/docker.parts/0445_autobuild.part.Dockerfile similarity index 100% rename from e2e/gzvulon__base_docker/docker.parts/0445_autobuild.part.Dockerfile rename to dockers/gzvulon__base_docker/docker.parts/0445_autobuild.part.Dockerfile diff --git a/e2e/gzvulon__base_docker/docker.scripts/0230_miniconda.part.Dockerfile b/dockers/gzvulon__base_docker/docker.scripts/0230_miniconda.part.Dockerfile similarity index 100% rename from e2e/gzvulon__base_docker/docker.scripts/0230_miniconda.part.Dockerfile rename to dockers/gzvulon__base_docker/docker.scripts/0230_miniconda.part.Dockerfile diff --git a/e2e/gzvulon__base_docker/docker.scripts/0230_miniconda_new.part.Dockerfile b/dockers/gzvulon__base_docker/docker.scripts/0230_miniconda_new.part.Dockerfile similarity index 100% rename from e2e/gzvulon__base_docker/docker.scripts/0230_miniconda_new.part.Dockerfile rename to dockers/gzvulon__base_docker/docker.scripts/0230_miniconda_new.part.Dockerfile diff --git a/e2e/gzvulon__base_docker/docker.scripts/python_23_conda__env_arg.part.run.sh b/dockers/gzvulon__base_docker/docker.scripts/python_23_conda__env_arg.part.run.sh similarity index 100% rename from e2e/gzvulon__base_docker/docker.scripts/python_23_conda__env_arg.part.run.sh rename to dockers/gzvulon__base_docker/docker.scripts/python_23_conda__env_arg.part.run.sh diff --git a/e2e/gzvulon__base_docker/generated/0230_miniconda.part.Dockerfile b/dockers/gzvulon__base_docker/generated/0230_miniconda.part.Dockerfile similarity index 100% rename from e2e/gzvulon__base_docker/generated/0230_miniconda.part.Dockerfile rename to dockers/gzvulon__base_docker/generated/0230_miniconda.part.Dockerfile diff --git a/e2e/gzvulon__base_docker/generated/0230_miniconda_new.part.Dockerfile b/dockers/gzvulon__base_docker/generated/0230_miniconda_new.part.Dockerfile similarity index 100% rename from e2e/gzvulon__base_docker/generated/0230_miniconda_new.part.Dockerfile rename to dockers/gzvulon__base_docker/generated/0230_miniconda_new.part.Dockerfile diff --git a/e2e/gzvulon__base_docker/generated/auto.Dockerfile b/dockers/gzvulon__base_docker/generated/auto.Dockerfile similarity index 100% rename from e2e/gzvulon__base_docker/generated/auto.Dockerfile rename to dockers/gzvulon__base_docker/generated/auto.Dockerfile diff --git a/e2e/gzvulon__base_docker/generated/default.build.sh b/dockers/gzvulon__base_docker/generated/default.build.sh similarity index 100% rename from e2e/gzvulon__base_docker/generated/default.build.sh rename to dockers/gzvulon__base_docker/generated/default.build.sh diff --git a/e2e/gzvulon__base_docker/generated/git.keep.txt b/dockers/gzvulon__base_docker/generated/git.keep.txt similarity index 100% rename from e2e/gzvulon__base_docker/generated/git.keep.txt rename to dockers/gzvulon__base_docker/generated/git.keep.txt diff --git a/e2e/gzvulon__base_docker/generated/install.dtasj.sh b/dockers/gzvulon__base_docker/generated/install.dtasj.sh similarity index 100% rename from e2e/gzvulon__base_docker/generated/install.dtasj.sh rename to dockers/gzvulon__base_docker/generated/install.dtasj.sh diff --git a/e2e/gzvulon__base_docker/generated/python_23_conda__env_arg.part.run.sh b/dockers/gzvulon__base_docker/generated/python_23_conda__env_arg.part.run.sh similarity index 100% rename from e2e/gzvulon__base_docker/generated/python_23_conda__env_arg.part.run.sh rename to dockers/gzvulon__base_docker/generated/python_23_conda__env_arg.part.run.sh diff --git a/e2e/gzvulon__base_docker/generated/run b/dockers/gzvulon__base_docker/generated/run similarity index 100% rename from e2e/gzvulon__base_docker/generated/run rename to dockers/gzvulon__base_docker/generated/run diff --git a/e2e/gzvulon__base_docker/generated/shell b/dockers/gzvulon__base_docker/generated/shell similarity index 100% rename from e2e/gzvulon__base_docker/generated/shell rename to dockers/gzvulon__base_docker/generated/shell diff --git a/e2e/gzvulon__base_docker/generated/task b/dockers/gzvulon__base_docker/generated/task similarity index 100% rename from e2e/gzvulon__base_docker/generated/task rename to dockers/gzvulon__base_docker/generated/task diff --git a/e2e/gzvulon__base_docker/tests/Taskfile.yml b/dockers/gzvulon__base_docker/tests/Taskfile.yml similarity index 100% rename from e2e/gzvulon__base_docker/tests/Taskfile.yml rename to dockers/gzvulon__base_docker/tests/Taskfile.yml diff --git a/e2e/gzvulon__base_docker/tests/repo.python-fire.all.reqs.txt b/dockers/gzvulon__base_docker/tests/repo.python-fire.all.reqs.txt similarity index 100% rename from e2e/gzvulon__base_docker/tests/repo.python-fire.all.reqs.txt rename to dockers/gzvulon__base_docker/tests/repo.python-fire.all.reqs.txt diff --git a/e2e/gzvulon__base_docker/tools/default.build.sh b/dockers/gzvulon__base_docker/tools/default.build.sh similarity index 100% rename from e2e/gzvulon__base_docker/tools/default.build.sh rename to dockers/gzvulon__base_docker/tools/default.build.sh diff --git a/e2e/gzvulon__base_docker/tools/install.dtasj.sh b/dockers/gzvulon__base_docker/tools/install.dtasj.sh similarity index 100% rename from e2e/gzvulon__base_docker/tools/install.dtasj.sh rename to dockers/gzvulon__base_docker/tools/install.dtasj.sh diff --git a/e2e/gzvulon__base_docker/tools/run b/dockers/gzvulon__base_docker/tools/run similarity index 100% rename from e2e/gzvulon__base_docker/tools/run rename to dockers/gzvulon__base_docker/tools/run diff --git a/e2e/gzvulon__base_docker/tools/shell b/dockers/gzvulon__base_docker/tools/shell similarity index 100% rename from e2e/gzvulon__base_docker/tools/shell rename to dockers/gzvulon__base_docker/tools/shell diff --git a/e2e/gzvulon__base_docker/tools/task b/dockers/gzvulon__base_docker/tools/task similarity index 100% rename from e2e/gzvulon__base_docker/tools/task rename to dockers/gzvulon__base_docker/tools/task diff --git a/dockers/jendev/Taskfile.yml b/dockers/jendev/Taskfile.yml new file mode 100644 index 0000000..1f07a56 --- /dev/null +++ b/dockers/jendev/Taskfile.yml @@ -0,0 +1,46 @@ +# https://taskfile.dev + +version: '3' + +vars: + GREETING: Hello, World! + +tasks: + default: + cmds: + - echo "{{.GREETING}}" + silent: true + + info: + desc: Some Info + cmds: + - echo "Reach Jenkins. http://localhost:12080" + - echo "Reach DevLab. http://localhost:12088" + silent: true + + build: + cmds: + - docker-compose build + stop: + cmds: + - docker-compose down || true + + run: + cmds: + - docker-compose up -d + + rerun: + cmds: + - task: stop + - docker-compose rm + - docker-compose up -d + + renew: + cmds: + - task: stop + - task: build + - task: rerun + - task: info + + + diff --git a/dockers/jendev/deps/default-test-user.groovy b/dockers/jendev/deps/default-test-user.groovy new file mode 100644 index 0000000..7bb8a46 --- /dev/null +++ b/dockers/jendev/deps/default-test-user.groovy @@ -0,0 +1,17 @@ +#!groovy + +import jenkins.model.* +import hudson.security.* +import jenkins.security.s2m.AdminWhitelistRule + +def instance = Jenkins.getInstance() + +def hudsonRealm = new HudsonPrivateSecurityRealm(false) +hudsonRealm.createAccount("tester", "tester") +instance.setSecurityRealm(hudsonRealm) + +def strategy = new FullControlOnceLoggedInAuthorizationStrategy() +instance.setAuthorizationStrategy(strategy) +instance.save() + +Jenkins.instance.getInjector().getInstance(AdminWhitelistRule.class).setMasterKillSwitch(false) \ No newline at end of file diff --git a/dockers/jendev/deps/jenkins.yaml b/dockers/jendev/deps/jenkins.yaml new file mode 100644 index 0000000..839eb28 --- /dev/null +++ b/dockers/jendev/deps/jenkins.yaml @@ -0,0 +1,130 @@ +jenkins: + agentProtocols: + - "JNLP4-connect" + - "Ping" + crumbIssuer: + standard: + excludeClientIPFromCrumb: true + disableRememberMe: false + labelAtoms: + - name: "master" + markupFormatter: "plainText" + mode: NORMAL + numExecutors: 2 + projectNamingStrategy: "standard" + quietPeriod: 5 + remotingSecurity: + enabled: false + scmCheckoutRetryCount: 0 + securityRealm: + local: + allowsSignup: true + enableCaptcha: false + users: + - id: "master" + name: "master" + properties: + - "apiToken" + - favoriting: + autofavoriteEnabled: true + - "favorite" + - preferredProvider: + providerId: "default" + - "timezone" + - mailer: + emailAddress: "master@master.master" + slaveAgentPort: 50000 + updateCenter: + sites: + - id: "default" + url: "https://updates.jenkins.io/update-center.json" +security: + apiToken: + creationOfLegacyTokenEnabled: true + tokenGenerationOnCreationEnabled: true + usageStatisticsEnabled: true + envInject: + enableLoadingFromMaster: false + enablePermissions: false + hideInjectedVars: false + globalJobDslSecurityConfiguration: + useScriptSecurity: false + sSHD: + port: -1 +unclassified: + artifactoryBuilder: + jfrogPipelinesServer: + bypassProxy: false + connectionRetries: 3 + credentialsConfig: + overridingCredentials: false + username: "****" + timeout: 300 + useCredentialsPlugin: false + bitbucketEndpointConfiguration: + endpoints: + - bitbucketCloudEndpoint: + enableCache: false + manageHooks: false + repositoriesCacheDuration: 0 + teamCacheDuration: 0 + buildDiscarders: + configuredBuildDiscarders: + - "jobBuildDiscarder" + buildTimestamp: + enableBuildTimestamp: true + pattern: "yyyy-MM-dd HH:mm:ss z" + timezone: "Etc/UTC" + fingerprints: + fingerprintCleanupDisabled: false + storage: "file" + gitHubConfiguration: + apiRateLimitChecker: ThrottleForNormalize + gitHubPluginConfig: + hookUrl: "http://localhost:8080/github-webhook/" + gitLabConnectionConfig: + connections: + - clientBuilderId: "autodetect" + connectionTimeout: 10 + ignoreCertificateErrors: false + readTimeout: 10 + gitLabServers: + servers: + - name: "default" + serverUrl: "https://gitlab.com" + gitSCM: + addGitTagAction: false + allowSecondFetch: false + createAccountBasedOnEmail: false + disableGitToolChooser: false + hideCredentials: false + showEntireCommitSummaryInChanges: false + useExistingAccountWithSameEmail: false + ivyBuildTrigger: + extendedVersionMatching: false + junitTestResultStorage: + storage: "file" + location: + adminAddress: "address not configured yet " + mailer: + charset: "UTF-8" + useSsl: false + useTls: false + mavenModuleSet: + localRepository: "default" + pluginImpl: + enableCredentialsFromNode: false + pollSCM: + pollingThreadCount: 10 + timestamper: + allPipelines: false + elapsedTimeFormat: "''HH:mm:ss.S' '" + systemTimeFormat: "''HH:mm:ss' '" +tool: + git: + installations: + - home: "git" + name: "Default" + mavenGlobalConfig: + globalSettingsProvider: "standard" + settingsProvider: "standard" diff --git a/dockers/jendev/deps/pip.deps.list.txt b/dockers/jendev/deps/pip.deps.list.txt new file mode 100644 index 0000000..d0bf88a --- /dev/null +++ b/dockers/jendev/deps/pip.deps.list.txt @@ -0,0 +1,9 @@ +fire +loguru +python-jenkins +python-gitlab +waiting +coverage +pytest +pytest-cov +pytest-xdist diff --git a/dockers/jendev/deps/tasks.vscode.deps.yml b/dockers/jendev/deps/tasks.vscode.deps.yml new file mode 100755 index 0000000..fa2c1d7 --- /dev/null +++ b/dockers/jendev/deps/tasks.vscode.deps.yml @@ -0,0 +1,88 @@ +#!/usr/local/bin/task --taskfile + +# https://taskfile.dev + +version: '3' + +vars: &_ref_vars + TG_NAME: VS Code Installations + _home_: "$(realpath ~)" + vsc_ext_dir: "{{._home_}}/.local/share/code-server/extensions" + + # bsdtar + + DEPS_APT: >- + libarchive-tools + curl +env: *_ref_vars + +tasks: + install: + desc: install this deps + cmds: + - task: install:vscode:full + + install:vscode:full: + desc: Install Zsh + dir: "$HOME" + cmds: + - task: _install:vscode:deps:apt + - task: _install:vscode:app:latest + - task: _install:vscode:plugins + + _install:vscode:deps:apt: + desc: _ + dir: "$HOME" + cmds: + - | + if [[ "${IN_DOCKER_BUILD}" == "1" ]]; then + sudo apt update + fi + - sudo apt-get install -y {{.DEPS_APT}} + status: + - echo '{{.DEPS_APT}}' | xargs which + + _install:vscode:app:latest: + desc: _ + dir: "$HOME" + cmds: + - curl -fsSL https://code-server.dev/install.sh | sh + status: + - which code-server + + _install:vscode:plugins: + desc: _ + dir: "$HOME" + cmds: + - task: _install:vscode:plugin:one + vars: + rpath: vscode-python/releases/download/2020.5.86806/ms-python-release.vsix + tname: ms-python.python-vscode-2.5.8 + + _install:vscode:plugin:one: + desc: _ + dir: /tmp + args: + rpath: vscode-python/releases/download/2020.5.86806/ms-python-release.vsix + tname: ms-python.python-vscode-2.5.8 + cmds: + - mkdir -p {{.vsc_ext_dir}} + - curl -JL https://github.com/Microsoft/{{.rpath}} | bsdtar -xvf - extension + - mv extension {{.vsc_ext_dir}}/{{.tname}} + status: + - test -e {{.vsc_ext_dir}}/{{.tname}} + +# https://github.com/cdr/code-server/issues/2341 +# code-server --install-extension (jupyter extension) --force + +# RUN mkdir -p ~/.local/lib ~/.local/bin +# RUN curl -fL https://github.com/cdr/code-server/releases/download/v3.7.4/code-server-3.7.4-linux-amd64.tar.gz \ +# | tar -C ~/.local/lib -xz +# RUN mv ~/.local/lib/code-server-3.7.4-linux-amd64 ~/.local/lib/code-server-3.7.4 +# RUN ln -s ~/.local/lib/code-server-3.7.4/bin/code-server ~/.local/bin/code-server +# RUN PATH="~/.local/bin:$PATH" + +# # Fix broken python plugin # https://github.com/cdr/code-server/issues/2341 +# RUN mkdir -p ~/.local/share/code-server/ && mkdir -p ~/.local/share/code-server/User && echo "{\"extensions.autoCheckUpdates\": false, \"extensions.autoUpdate\": false}" > ~/.local/share/code-server/User/settings.json +# RUN wget https://github.com/microsoft/vscode-python/releases/download/2020.10.332292344/ms-python-release.vsix \ +# && ~/.local/bin/code-server --install-extension ./ms-python-release.vsix || true \ No newline at end of file diff --git a/dockers/jendev/docker-compose.yml b/dockers/jendev/docker-compose.yml new file mode 100644 index 0000000..fcfb5b3 --- /dev/null +++ b/dockers/jendev/docker-compose.yml @@ -0,0 +1,43 @@ +# https://stackoverflow.com/questions/48379107/how-can-i-ping-other-containers-in-a-docker-network-through-their-hostname + +version: "3" +services: + jencondadev.samexample.com: + hostname: 'jencondadev.samexample.com' + image: jencondadev + build: + context: . + dockerfile: jencondadev.Dockerfile + + # build: jenconda + ports: + - "5000" + - "12080:8080" + - "50000" + volumes: + - "/var/run/docker.sock:/var/run/docker.sock" + - "../..:/repo" + + # network_mode: "host" + networks: + ntjendev: +# https://www.sumologic.com/blog/how-to-build-applications-docker-compose/ + jupcondadev.samexample.com: + image: jupconda_img + hostname: 'jupcondadev.samexample.com' + build: + context: . + dockerfile: jupconda.Dockerfile + # args: + # buildno: 1 + ports: + - "12088:8888" + volumes: + # - "jupconda.samexample.com" + - "../..:/home/jovyan/repo" + networks: + ntjendev: + +networks: + ntjendev: + driver: bridge diff --git a/dockers/jendev/jencondadev.Dockerfile b/dockers/jendev/jencondadev.Dockerfile new file mode 100644 index 0000000..7639ce8 --- /dev/null +++ b/dockers/jendev/jencondadev.Dockerfile @@ -0,0 +1,13 @@ +ARG BASE_CONTAINER=jenconda +FROM $BASE_CONTAINER + +COPY deps/jenkins.yaml /usr/share/jenkins/jenkins.yaml +COPY deps/jenkins.yaml /usr/share/jenkins/ref/jenkins.yaml + +COPY deps/default-test-user.groovy /usr/share/jenkins/ref/init.groovy.d/ + + +# RUN cd /tmp \ +# && curl -O http://127.0.0.1:8080/jnlpJars/jenkins-cli.jar \ +# && echo 'jenkins.model.Jenkins.instance.securityRealm.createAccount("user1", "password123")' \ +# | java -jar jenkins-cli.jar -s http://localhost:8080 groovy = diff --git a/dockers/jendev/jupconda.Dockerfile b/dockers/jendev/jupconda.Dockerfile new file mode 100644 index 0000000..d6787bf --- /dev/null +++ b/dockers/jendev/jupconda.Dockerfile @@ -0,0 +1,48 @@ +ARG BASE_CONTAINER=jupyter/pyspark-notebook +FROM $BASE_CONTAINER + +# Install task rclone yq jq +# https://taskfile.dev/#/installation +USER root +RUN apt update && apt install -y curl +RUN sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin +RUN echo "${NB_USER} ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers + +USER $NB_USER +ENV IN_DOCKER_BUILD 1 +# Install VS Code +# ------@@stage=prephook section=task.docker.cmds task=mxsh:install:zsh:kubeflow +COPY deps/tasks.vscode.deps.yml deps/tasks.vscode.deps.yml +RUN task -t deps/tasks.vscode.deps.yml install + +# Install jupyter proxies for VS Code and RStudio +RUN pip install --no-cache-dir jupyter-server-proxy && \ + pip install --no-cache-dir jupyter-vscode-proxy + +RUN jupyter labextension install @jupyterlab/server-proxy + + +ENV PATH /home/${NB_USER}/.asdf/bin:/home/${NB_USER}/.asdf/shims:$PATH +RUN /bin/bash -c "git clone https://github.com/asdf-vm/asdf.git ~/.asdf --branch v0.8.0 && \ + asdf plugin-add groovy https://github.com/weibemoura/asdf-groovy.git && \ + asdf install groovy 3.0.7 && \ + asdf global groovy 3.0.7" +RUN \ + echo -e '\n. $HOME/.asdf/asdf.sh' >> ~/.bashrc \ + && echo -e '\n. $HOME/.asdf/completions/asdf.bash' >> ~/.bashrc +# elixir +COPY deps/pip.deps.list.txt deps/pip.deps.list.txt +RUN pip install -r deps/pip.deps.list.txt +# --- finishes --- + +ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 + +EXPOSE 8888 + +WORKDIR ${HOME} +# mount point for kubeflow +VOLUME /home/jovyan + +# Configure container startup +ENV NB_PREFIX / +CMD ["sh","-c", "jupyter lab --notebook-dir=/home/jovyan --ip=0.0.0.0 --no-browser --allow-root --port=8888 --NotebookApp.token='' --NotebookApp.password='' --NotebookApp.allow_origin='*' --NotebookApp.base_url=${NB_PREFIX}"] diff --git a/dockers/jenkins-master/.dockerignore b/dockers/jenkins-master/.dockerignore new file mode 100644 index 0000000..e815b6f --- /dev/null +++ b/dockers/jenkins-master/.dockerignore @@ -0,0 +1 @@ +GITLAB_HOME diff --git a/e2e/jenkins-master/.gitignore b/dockers/jenkins-master/.gitignore similarity index 56% rename from e2e/jenkins-master/.gitignore rename to dockers/jenkins-master/.gitignore index 08c76a1..a6874a3 100644 --- a/e2e/jenkins-master/.gitignore +++ b/dockers/jenkins-master/.gitignore @@ -1,2 +1,3 @@ .vscode .task +GITLAB_HOME \ No newline at end of file diff --git a/e2e/jenkins-master/Dockerfile b/dockers/jenkins-master/Dockerfile similarity index 55% rename from e2e/jenkins-master/Dockerfile rename to dockers/jenkins-master/Dockerfile index 970b575..1101d7f 100755 --- a/e2e/jenkins-master/Dockerfile +++ b/dockers/jenkins-master/Dockerfile @@ -1,4 +1,4 @@ -FROM jenkins/jenkins:2.237 +FROM jenkins/jenkins:2.272 ENV JENKINS_USER admin ENV JENKINS_PASS admin @@ -22,16 +22,36 @@ RUN curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-c # added ny yairvd COPY install.scripts/install_taskgo.sh install.scripts/install_taskgo.sh -RUN chmod +x install.scripts/install_taskgo.sh \ - && install.scripts/install_taskgo.sh +RUN chmod +x install.scripts/install_taskgo.sh +RUN install.scripts/install_taskgo.sh + +RUN cd /tmp \ + && wget https://github.com/mikefarah/yq/releases/download/v4.2.0/yq_linux_amd64 \ + && mv yq_linux_amd64 /usr/bin/yq + +RUN cd /tmp \ + && wget https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 \ + && mv jq-linux64 /usr/bin/jq USER jenkins COPY plugins.txt /usr/share/jenkins/plugins.txt -RUN /usr/local/bin/install-plugins.sh < /usr/share/jenkins/plugins.txt +RUN jenkins-plugin-cli -f /usr/share/jenkins/plugins.txt + +# COPY jenkins.yaml /usr/share/jenkins/jenkins.yaml +# COPY jenkins.yaml /usr/share/jenkins/ref/jenkins.yaml +COPY jenkins.yaml /usr/share/stuff/jenkins.yaml + + +# COPY plugins_manual.txt /usr/share/jenkins/ref/plugins_manual.txt + +# ENV JENKINS_UC_DOWNLOAD=${JENKINS_UC_EXPERIMENTAL}/download +# RUN jenkins-plugin-cli -f /usr/share/jenkins/ref/plugins_manual.txt + + -COPY default-user.groovy /usr/share/jenkins/ref/init.groovy.d/ -COPY job_sys /usr/share/jenkins/job_sys +COPY default-newci-job.groovy /usr/share/jenkins/ref/init.groovy.d/ +COPY jobs_sys /usr/share/jenkins/job_sys # COPY custom.groovy /usr/share/jenkins/ref/init.groovy.d/custom.groovy # https://medium.com/the-devops-ship/custom-jenkins-dockerfile-jenkins-docker-image-with-pre-installed-plugins-default-admin-user-d0107b582577 diff --git a/e2e/jenkins-master/Dockerfile-alpine b/dockers/jenkins-master/Dockerfile-alpine similarity index 100% rename from e2e/jenkins-master/Dockerfile-alpine rename to dockers/jenkins-master/Dockerfile-alpine diff --git a/e2e/jenkins-master/README.md b/dockers/jenkins-master/README.md similarity index 100% rename from e2e/jenkins-master/README.md rename to dockers/jenkins-master/README.md diff --git a/dockers/jenkins-master/Taskfile copy.yml b/dockers/jenkins-master/Taskfile copy.yml new file mode 100644 index 0000000..1fcd2b4 --- /dev/null +++ b/dockers/jenkins-master/Taskfile copy.yml @@ -0,0 +1,209 @@ +# https://taskfile.dev + +version: '3' + +vars: + GREETING: JenkinsDockers + +tasks: + default: + cmds: + - echo "{{.GREETING}}" + - task -l + silent: true + + docker:build:jenmaster: + desc: _ + cmds: + - docker build . -t jenmaster + + docker:build:jenconda: + desc: _ + deps: + - docker:build:jenmaster + cmds: + - docker build -f install.scripts/install-conda-in-docker.Dockerfile -t jenconda install.scripts + + docker:build: + desc: _ + deps: + - docker:build:jenconda + + # docker:run:prod: + # desc: _ + # cmds: + # - docker rm -f jenconda_c || true + # - test -d /opt/dockers-data || sudo mkdir -p /opt/dockers-data && sudo chmod -R 777 /opt/dockers-data + # - test -d /opt/dockers-data/jenkins_home || sudo mkdir -p /opt/dockers-data/jenkins_home && sudo chmod -R 777 /opt/dockers-data/jenkins_home + # - | + # docker run \ + # --name jenconda_c \ + # -d \ + # -p 8080:8080 \ + # -p 50000:50000 \ + # -v /opt/dockers-data/jenkins_home:/var/jenkins_home:rw \ + # -v $(realpath ~)/.ssh:/var/jenkins_home/.ssh \ + # -v /var/run/docker.sock:/var/run/docker.sock \ + # jenconda + # -v $(realpath ~)/.ssh:/var/jenkins_home/.ssh \ + # -v $(realpath ~)/.aws:/var/jenkins_home/.aws \ + docker:run:prod-me: + desc: _ + cmds: + - docker rm -f jenconda_c || true + - mkdir -p $(realpath ~)/shared/jenkins_c + - test -d /data/dockers_data || mkdir -p /data/dockers_data && chmod -R 777 /data/dockers_data || echo oy + - test -d /data/dockers_data/jenkins_home || mkdir -p /data/dockers_data/jenkins_home && chmod -R 777 /data/dockers_data/jenkins_home || echo ai + - | + docker run \ + --name jenconda_c \ + -d \ + -p 8080:8080 \ + -p 50000:50000 \ + -p 5000:5000 \ + -v /data/dockers_data/jenkins_home:/var/jenkins_home:rw \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v $(realpath ~)/shared/jenkins_c:/shared/jenkins_c \ + jenconda:0.0.1 + + docker:run:local: + desc: _ + cmds: + - docker rm -f jenconda_lc || true + - mkdir -p $(realpath ~)/shared/jenkins_c + # -v /data/dockers_data/jenkins_home:/var/jenkins_home:rw \ + # - test -d /data/dockers_data || mkdir -p /data/dockers_data && chmod -R 777 /data/dockers_data || echo oy + # - test -d /data/dockers_data/jenkins_home || mkdir -p /data/dockers_data/jenkins_home && chmod -R 777 /data/dockers_data/jenkins_home || echo ai + - | + docker run \ + --name jenconda_lc \ + -d \ + -p 8080:8080 \ + -p 50000:50000 \ + -p 5000:5000 \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v $(realpath ~)/shared/jenkins_c:/shared/jenkins_c \ + jenconda + + docker:run:dev: + desc: _ + cmds: + - docker rm -f jenconda_c_dev || true + - test -d /opt/dockers-data || (sudo mkdir -p /opt/dockers-data && (sudo chmod -R 777 /opt/dockers-data || true)) + - test -d /opt/dockers-data/jenkins_home_dev || (sudo mkdir -p /opt/dockers-data/jenkins_home_dev && (sudo chmod -R 777 /opt/dockers-data/jenkins_home_dev || true)) + - | + docker run \ + --name jenconda_c_dev \ + -d \ + -p 8830:8080 \ + -p 58300:50000 \ + -v /opt/dockers-data/jenkins_home_dev:/var/jenkins_home:rw \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v $(realpath ~):/var/userhome:ro \ + -v $(realpath ../..):/var/jenlib_repo:ro \ + jenconda + + docker:run:test:init: + desc: _ + vars: + VOLUME_NAME: {sh: echo "jenconda_v_dtest"} + DEMONIZE: -d + cmds: + # - docker volume rm {{.VOLUME_NAME}}|| true + - docker rm -f jenconda_c_dtest || true + - docker volume create {{.VOLUME_NAME}} || true + # -d \ + - | + docker run \ + --name jenconda_c_dtest \ + -d \ + -p 18808:8080 \ + -p 18500:50000 \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v {{.VOLUME_NAME}}:/var/jenkins_home:rw \ + -v $(realpath ~):/var/userhome:ro \ + -v $(realpath ../..):/var/jenlib_repo:ro \ + jenconda + - ./wait-for-it.sh localhost:18808 + - echo access "http://localhost:18808" + + docker:run:test:flow: + desc: _ + cmds: + - ./wait-for-it.sh localhost:18808 + + docker:run:test:full: + desc: _ + cmds: + - task docker:run:test:init + - task docker:run:test:flow + + ci-flow: + desc: _ + cmds: + - task docker:build + - task docker:run:test:full + + + docker:clean:data: + desc: _ + cmds: + - rm -rf ~/jenkins_home/* ~/jenkins_home/.* + + install:jcli: + desc: _ + cmds: + - which brew && brew install jcli || true + - | + which brew \ + || curl -L https://github.com/jenkins-zh/jenkins-cli/releases/latest/download/jcli-linux-amd64.tar.gz|tar xzv \ + && sudo mv jcli /usr/local/bin/ + + shapshot:update: + desc: _ + cmds: + - jcli plugin list > install.snapshots/snapshot_2020-05-20--00-40-45/jcli_plugin_list.org.tsv + + dm:sync: + desc: _ + vars: + jentest_machine: {sh: cat ./e2e/cfg/_machine.name.txt} + cmds: + - | + docker-machine scp -r . {{.jentest_machine}}:/home/docker/_jm + + dm:ssh: + desc: _ + vars: + jentest_machine: {sh: cat ./e2e/cfg/_machine.name.txt} + cmds: + - | + docker-machine ssh {{.jentest_machine}} + + show-plugins-from: + desc: _ + vars: + target: install.snapshots/snapshot_2020-05-20--00-40-45 + cmds: + # - | + # cat "{{.target}}/jcli_plugin_list.org.tsv" \ + # | python -c " + # import fileinput + # for line in fileinput.input():print(':'.join(line.split()[:2])) + # " + - | + cat "{{.target}}/jcli_plugin_list.org.tsv" \ + | python -c "import fileinput; \ + set(map( \ + lambda x: print(':'.join(x.split()[:2])), \ + list(fileinput.input())[1:])) \ + " + silent: yes + + gen-plugins-from: + desc: _ + cmds: + - task show-plugins-from > plugins.txt 2>&1 + + # - cat "{{.target}}/jcli_plugin_list.org.tsv" | sed 's/ false/blas/' + # - cat "{{.target}}/jcli_plugin_list.org.tsv" | python -c "print(x.split()" diff --git a/e2e/jenkins-master/Taskfile.yml b/dockers/jenkins-master/Taskfile.yml similarity index 66% rename from e2e/jenkins-master/Taskfile.yml rename to dockers/jenkins-master/Taskfile.yml index f92130c..f7b2857 100644 --- a/e2e/jenkins-master/Taskfile.yml +++ b/dockers/jenkins-master/Taskfile.yml @@ -1,9 +1,10 @@ # https://taskfile.dev -version: '2' +version: '3' vars: GREETING: JenkinsDockers + jentest_machine: jentest tasks: default: @@ -28,29 +29,69 @@ tasks: desc: _ deps: - docker:build:jenconda + build: + desc: _ + deps: + - docker:build - docker:run:prod: + spin:up:bash: desc: _ cmds: - - docker rm -f jenconda_c || true - - test -d /opt/dockers-data || sudo mkdir -p /opt/dockers-data && sudo chmod -R 777 /opt/dockers-data - - test -d /opt/dockers-data/jenkins_home || sudo mkdir -p /opt/dockers-data/jenkins_home && sudo chmod -R 777 /opt/dockers-data/jenkins_home + - docker rm -f jenconda_lc || true + - mkdir -p $(realpath ~)/shared/jenkins_c - | docker run \ - --name jenconda_c \ + --name jenconda_lc \ -d \ - -p 8880:8080 \ - -p 58000:50000 \ - -v /opt/dockers-data/jenkins_home:/var/jenkins_home:rw \ + -p 8080:8080 \ + -p 50000:50000 \ + -p 5000:5000 \ -v /var/run/docker.sock:/var/run/docker.sock \ + -v $(realpath ~)/shared/jenkins_c:/shared/jenkins_c \ + jenconda + + spin:up: + desc: _ + cmds: + - mkdir -p /tmp/GITLAB_HOME + - chmod 777 /tmp/GITLAB_HOME + - docker-compose up -d + + spin:up:dev: + desc: _ + cmds: + - docker-compose up -d + + status: + desc: _ + cmds: + - docker-compose ps + + docker:run:local: + desc: _ + cmds: + - docker rm -f jenconda_lc || true + - mkdir -p $(realpath ~)/shared/jenkins_c + # -v /data/dockers_data/jenkins_home:/var/jenkins_home:rw \ + # - test -d /data/dockers_data || mkdir -p /data/dockers_data && chmod -R 777 /data/dockers_data || echo oy + # - test -d /data/dockers_data/jenkins_home || mkdir -p /data/dockers_data/jenkins_home && chmod -R 777 /data/dockers_data/jenkins_home || echo ai + - | + docker run \ + --name jenconda_lc \ + -d \ + -p 8080:8080 \ + -p 50000:50000 \ + -p 5000:5000 \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v $(realpath ~)/shared/jenkins_c:/shared/jenkins_c \ jenconda docker:run:dev: desc: _ cmds: - docker rm -f jenconda_c_dev || true - - test -d /opt/dockers-data || (sudo mkdir -p /opt/dockers-data && sudo chmod -R 777 /opt/dockers-data) - - test -d /opt/dockers-data/jenkins_home_dev || (sudo mkdir -p /opt/dockers-data/jenkins_home_dev && sudo chmod -R 777 /opt/dockers-data/jenkins_home_dev) + - test -d /opt/dockers-data || (sudo mkdir -p /opt/dockers-data && (sudo chmod -R 777 /opt/dockers-data || true)) + - test -d /opt/dockers-data/jenkins_home_dev || (sudo mkdir -p /opt/dockers-data/jenkins_home_dev && (sudo chmod -R 777 /opt/dockers-data/jenkins_home_dev || true)) - | docker run \ --name jenconda_c_dev \ @@ -98,7 +139,11 @@ tasks: - task docker:run:test:init - task docker:run:test:flow - + ci-flow: + desc: _ + cmds: + - task docker:build + - task docker:run:test:full docker:clean:data: @@ -107,7 +152,7 @@ tasks: - rm -rf ~/jenkins_home/* ~/jenkins_home/.* install:jcli: - desc: _ + # desc: _ cmds: - which brew && brew install jcli || true - | @@ -122,16 +167,12 @@ tasks: dm:sync: desc: _ - vars: - jentest_machine: {sh: cat e2e/cfg/_machine.name.txt} cmds: - | docker-machine scp -r . {{.jentest_machine}}:/home/docker/_jm dm:ssh: desc: _ - vars: - jentest_machine: {sh: cat e2e/cfg/_machine.name.txt} cmds: - | docker-machine ssh {{.jentest_machine}} @@ -162,4 +203,15 @@ tasks: - task show-plugins-from > plugins.txt 2>&1 # - cat "{{.target}}/jcli_plugin_list.org.tsv" | sed 's/ false/blas/' - # - cat "{{.target}}/jcli_plugin_list.org.tsv" | python -c "print(x.split()" \ No newline at end of file + # - cat "{{.target}}/jcli_plugin_list.org.tsv" | python -c "print(x.split()" + + dm:list: + desc: _ + cmds: + - docker-machine ls + + tpa: + desc: _ + cmds: + - curl http://192.168.99.100:8880/ + - curl http://192.168.99.100:8880/ | grep 'Please wait while Jenkins is getting ready to work' diff --git a/e2e/jenkins-master/_metarepo/jenkins.rubbin.github.docker.repo.yml b/dockers/jenkins-master/_metarepo/jenkins.rubbin.github.docker.repo.yml similarity index 100% rename from e2e/jenkins-master/_metarepo/jenkins.rubbin.github.docker.repo.yml rename to dockers/jenkins-master/_metarepo/jenkins.rubbin.github.docker.repo.yml diff --git a/dockers/jenkins-master/cicd/jenBuild.groovy b/dockers/jenkins-master/cicd/jenBuild.groovy new file mode 100644 index 0000000..0857c77 --- /dev/null +++ b/dockers/jenkins-master/cicd/jenBuild.groovy @@ -0,0 +1,39 @@ +@Library("Jenkins_Jenlib@master") _ +// ===== /SHARED PART ==== + +// --- globas vars --- +def jg = [ + scmvars: null, + cmds: [], + tstages: [:], + prj_dir: '.', + commits: [], + params: [ + samle: 'yes' + ] +] + +node() {timestamps { + + stage('fetch'){ + echo 'Fetch source' + def scmvars = checkout scm + def more = "repo_validate" + jen.set_build_name(currentBuild, scmvars, more) + jg['scmvars'] = scmvars + } + + stage('commits'){ + jen.desc_from_commits(currentBuild, jg) + } + + catchError { dir('dockers/jenkins-master'){ + jen.step_stages_from_tasks( + jg, jg.prj_dir, + 'Taskfile.yml', 'ci-flow') + }} + + stage('finish'){ + echo "done" + } +}} diff --git a/e2e/jenkins-master/default-user.groovy b/dockers/jenkins-master/default-newci-job.groovy similarity index 100% rename from e2e/jenkins-master/default-user.groovy rename to dockers/jenkins-master/default-newci-job.groovy diff --git a/dockers/jenkins-master/docker-compose.yml b/dockers/jenkins-master/docker-compose.yml new file mode 100644 index 0000000..0a24d83 --- /dev/null +++ b/dockers/jenkins-master/docker-compose.yml @@ -0,0 +1,54 @@ +# https://stackoverflow.com/questions/48379107/how-can-i-ping-other-containers-in-a-docker-network-through-their-hostname + +version: "3" +services: + jencondait.samexample.com: + hostname: 'jencondait.samexample.com' + # build: jenconda + image: jenconda + ports: + - "5000:5000" + - "8080:8080" + - "50000:50000" + volumes: + - "/var/run/docker.sock:/var/run/docker.sock" + - "../..:/repo" + # network_mode: "host" + networks: + test: + + httpit: + image: pierrezemb/gostatic + # ports: + # - "8043:8043" + volumes: + - "../..:/srv/http" + networks: + test: + + gitlabit.samexample.com: + # 10.8.1 12.1.6 13.2.10 + image: 'gitlab/gitlab-ce:12.1.6-ce.0' + restart: always + # https://gitlab.example.com/ + # gitlab_rails['initial_root_password'] = 'qwe123' + hostname: 'gitlabit.samexample.com' + environment: + GITLAB_OMNIBUS_CONFIG: | + external_url 'http://gitlabit.samexample.com:33080' + gitlab_rails['gitlab_shell_ssh_port'] = 33022 + gitlab_rails['initial_root_password'] = 'qwe123qwe123' + ports: + - '33080:33080' + - '33443:443' + - '33022:22' + volumes: + - '/tmp/GITLAB_HOME/config:/etc/gitlab' + - '/tmp/GITLAB_HOME/logs:/var/log/gitlab' + - '/tmp/GITLAB_HOME/data:/var/opt/gitlab' + networks: + test: + +networks: + test: + driver: bridge \ No newline at end of file diff --git a/e2e/jenkins-master/init.groovy.d/47_shared-pipeline-lib.groovy b/dockers/jenkins-master/init.groovy.d/47_shared-pipeline-lib.groovy similarity index 100% rename from e2e/jenkins-master/init.groovy.d/47_shared-pipeline-lib.groovy rename to dockers/jenkins-master/init.groovy.d/47_shared-pipeline-lib.groovy diff --git a/e2e/jenkins-master/init.groovy.d/50_load-job-dsl-scripts.groovy b/dockers/jenkins-master/init.groovy.d/50_load-job-dsl-scripts.groovy similarity index 100% rename from e2e/jenkins-master/init.groovy.d/50_load-job-dsl-scripts.groovy rename to dockers/jenkins-master/init.groovy.d/50_load-job-dsl-scripts.groovy diff --git a/e2e/jenkins-master/install.dtask/.gitignore b/dockers/jenkins-master/install.dtask/.gitignore similarity index 100% rename from e2e/jenkins-master/install.dtask/.gitignore rename to dockers/jenkins-master/install.dtask/.gitignore diff --git a/e2e/jenkins-master/install.dtask/README.md b/dockers/jenkins-master/install.dtask/README.md similarity index 100% rename from e2e/jenkins-master/install.dtask/README.md rename to dockers/jenkins-master/install.dtask/README.md diff --git a/e2e/jenkins-master/install.dtask/Taskfile.yml b/dockers/jenkins-master/install.dtask/Taskfile.yml similarity index 100% rename from e2e/jenkins-master/install.dtask/Taskfile.yml rename to dockers/jenkins-master/install.dtask/Taskfile.yml diff --git a/e2e/jenkins-master/install.dtask/assets/git.keep.txt b/dockers/jenkins-master/install.dtask/assets/git.keep.txt similarity index 100% rename from e2e/jenkins-master/install.dtask/assets/git.keep.txt rename to dockers/jenkins-master/install.dtask/assets/git.keep.txt diff --git a/e2e/jenkins-master/install.dtask/auto.Dockerfile b/dockers/jenkins-master/install.dtask/auto.Dockerfile similarity index 100% rename from e2e/jenkins-master/install.dtask/auto.Dockerfile rename to dockers/jenkins-master/install.dtask/auto.Dockerfile diff --git a/e2e/jenkins-master/install.dtask/auto.build.sh b/dockers/jenkins-master/install.dtask/auto.build.sh similarity index 100% rename from e2e/jenkins-master/install.dtask/auto.build.sh rename to dockers/jenkins-master/install.dtask/auto.build.sh diff --git a/e2e/jenkins-master/install.dtask/docker-client.part.Dockerfile b/dockers/jenkins-master/install.dtask/docker-client.part.Dockerfile similarity index 100% rename from e2e/jenkins-master/install.dtask/docker-client.part.Dockerfile rename to dockers/jenkins-master/install.dtask/docker-client.part.Dockerfile diff --git a/e2e/jenkins-master/install.dtask/docker.parts/git.keep.txt b/dockers/jenkins-master/install.dtask/docker.parts/git.keep.txt similarity index 100% rename from e2e/jenkins-master/install.dtask/docker.parts/git.keep.txt rename to dockers/jenkins-master/install.dtask/docker.parts/git.keep.txt diff --git a/e2e/jenkins-master/install.dtask/dtask.part.Dockerfile b/dockers/jenkins-master/install.dtask/dtask.part.Dockerfile similarity index 100% rename from e2e/jenkins-master/install.dtask/dtask.part.Dockerfile rename to dockers/jenkins-master/install.dtask/dtask.part.Dockerfile diff --git a/e2e/jenkins-master/install.dtask/generated/git.keep.txt b/dockers/jenkins-master/install.dtask/generated/git.keep.txt similarity index 100% rename from e2e/jenkins-master/install.dtask/generated/git.keep.txt rename to dockers/jenkins-master/install.dtask/generated/git.keep.txt diff --git a/e2e/jenkins-master/install.dtask/install.dtasj.sh b/dockers/jenkins-master/install.dtask/install.dtasj.sh similarity index 100% rename from e2e/jenkins-master/install.dtask/install.dtasj.sh rename to dockers/jenkins-master/install.dtask/install.dtasj.sh diff --git a/e2e/jenkins-master/install.dtask/miniconda.part.Dockerfile b/dockers/jenkins-master/install.dtask/miniconda.part.Dockerfile similarity index 100% rename from e2e/jenkins-master/install.dtask/miniconda.part.Dockerfile rename to dockers/jenkins-master/install.dtask/miniconda.part.Dockerfile diff --git a/e2e/jenkins-master/install.dtask/task b/dockers/jenkins-master/install.dtask/task similarity index 100% rename from e2e/jenkins-master/install.dtask/task rename to dockers/jenkins-master/install.dtask/task diff --git a/e2e/jenkins-master/install.scripts/install-conda-in-docker.Dockerfile b/dockers/jenkins-master/install.scripts/install-conda-in-docker.Dockerfile similarity index 100% rename from e2e/jenkins-master/install.scripts/install-conda-in-docker.Dockerfile rename to dockers/jenkins-master/install.scripts/install-conda-in-docker.Dockerfile diff --git a/e2e/jenkins-master/install.scripts/install_mc.sh b/dockers/jenkins-master/install.scripts/install_mc.sh similarity index 100% rename from e2e/jenkins-master/install.scripts/install_mc.sh rename to dockers/jenkins-master/install.scripts/install_mc.sh diff --git a/e2e/jenkins-master/install.scripts/install_taskgo.sh b/dockers/jenkins-master/install.scripts/install_taskgo.sh similarity index 93% rename from e2e/jenkins-master/install.scripts/install_taskgo.sh rename to dockers/jenkins-master/install.scripts/install_taskgo.sh index 329a134..e48f312 100755 --- a/e2e/jenkins-master/install.scripts/install_taskgo.sh +++ b/dockers/jenkins-master/install.scripts/install_taskgo.sh @@ -1,6 +1,6 @@ #! /bin/bash -TASKGO_VERSION=v2.8.0 +TASKGO_VERSION=v3.0.0 TASKGO_ZIPNAME=task_linux_amd64.tar.gz echo install task executable diff --git a/e2e/jenkins-master/install.snapshots/backup_2020-05-20--03-28-19/plugins.txt b/dockers/jenkins-master/install.snapshots/backup_2020-05-20--03-28-19/plugins.txt similarity index 100% rename from e2e/jenkins-master/install.snapshots/backup_2020-05-20--03-28-19/plugins.txt rename to dockers/jenkins-master/install.snapshots/backup_2020-05-20--03-28-19/plugins.txt diff --git a/e2e/jenkins-master/install.snapshots/backup_2020-05-20--03-28-19/plugins_org.txt b/dockers/jenkins-master/install.snapshots/backup_2020-05-20--03-28-19/plugins_org.txt similarity index 100% rename from e2e/jenkins-master/install.snapshots/backup_2020-05-20--03-28-19/plugins_org.txt rename to dockers/jenkins-master/install.snapshots/backup_2020-05-20--03-28-19/plugins_org.txt diff --git a/e2e/jenkins-master/install.snapshots/snapshot_2020-05-20--00-40-45/jcli_plugin_list.org.tsv b/dockers/jenkins-master/install.snapshots/snapshot_2020-05-20--00-40-45/jcli_plugin_list.org.tsv similarity index 100% rename from e2e/jenkins-master/install.snapshots/snapshot_2020-05-20--00-40-45/jcli_plugin_list.org.tsv rename to dockers/jenkins-master/install.snapshots/snapshot_2020-05-20--00-40-45/jcli_plugin_list.org.tsv diff --git a/e2e/jenkins-master/jen_console_scripts/create_job__script_console.groovy b/dockers/jenkins-master/jen_console_scripts/create_job__script_console.groovy similarity index 100% rename from e2e/jenkins-master/jen_console_scripts/create_job__script_console.groovy rename to dockers/jenkins-master/jen_console_scripts/create_job__script_console.groovy diff --git a/e2e/jenkins-master/jen_console_scripts/create_multiple_jobs.groovy b/dockers/jenkins-master/jen_console_scripts/create_multiple_jobs.groovy similarity index 100% rename from e2e/jenkins-master/jen_console_scripts/create_multiple_jobs.groovy rename to dockers/jenkins-master/jen_console_scripts/create_multiple_jobs.groovy diff --git a/e2e/jenkins-master/jen_dsl_init_jobs/README.md b/dockers/jenkins-master/jen_dsl_init_jobs/README.md similarity index 100% rename from e2e/jenkins-master/jen_dsl_init_jobs/README.md rename to dockers/jenkins-master/jen_dsl_init_jobs/README.md diff --git a/e2e/jenkins-master/jen_dsl_scripts/dsl_pipeline_example.dsl.groovy b/dockers/jenkins-master/jen_dsl_scripts/dsl_pipeline_example.dsl.groovy similarity index 100% rename from e2e/jenkins-master/jen_dsl_scripts/dsl_pipeline_example.dsl.groovy rename to dockers/jenkins-master/jen_dsl_scripts/dsl_pipeline_example.dsl.groovy diff --git a/e2e/jenkins-master/jen_dsl_scripts/init_job_dsl_longer.groovy b/dockers/jenkins-master/jen_dsl_scripts/init_job_dsl_longer.groovy similarity index 100% rename from e2e/jenkins-master/jen_dsl_scripts/init_job_dsl_longer.groovy rename to dockers/jenkins-master/jen_dsl_scripts/init_job_dsl_longer.groovy diff --git a/e2e/jenkins-master/jen_dsl_scripts/init_job_dsl_short.groovy b/dockers/jenkins-master/jen_dsl_scripts/init_job_dsl_short.groovy similarity index 100% rename from e2e/jenkins-master/jen_dsl_scripts/init_job_dsl_short.groovy rename to dockers/jenkins-master/jen_dsl_scripts/init_job_dsl_short.groovy diff --git a/e2e/jenkins-master/jen_dsl_scripts/jenlib_test.jdsl.groovy b/dockers/jenkins-master/jen_dsl_scripts/jenlib_test.jdsl.groovy similarity index 83% rename from e2e/jenkins-master/jen_dsl_scripts/jenlib_test.jdsl.groovy rename to dockers/jenkins-master/jen_dsl_scripts/jenlib_test.jdsl.groovy index 714b276..411767d 100644 --- a/e2e/jenkins-master/jen_dsl_scripts/jenlib_test.jdsl.groovy +++ b/dockers/jenkins-master/jen_dsl_scripts/jenlib_test.jdsl.groovy @@ -13,7 +13,7 @@ pipelineJob("jenlib_pipe_master_ci") { branch("*/master") } } - scriptPath("e2e/data4test/parallel_tasks_jen/build.groovy") + scriptPath("data4test/parallel_tasks_jen/build.groovy") } } disabled(false) @@ -37,7 +37,7 @@ task \ add \ jobname=jenlib_pipe_master_ci \ github_p=gzvulon/jenlib -script_p=e2e/data4test/parallel_tasks_jen/build.groovy +script_p=data4test/parallel_tasks_jen/build.groovy mkdir -p _projects_dsl ''' \ No newline at end of file diff --git a/e2e/jenkins-master/jen_dsl_scripts/jenlib_test.tmpl.jdsl.groovy b/dockers/jenkins-master/jen_dsl_scripts/jenlib_test.tmpl.jdsl.groovy similarity index 84% rename from e2e/jenkins-master/jen_dsl_scripts/jenlib_test.tmpl.jdsl.groovy rename to dockers/jenkins-master/jen_dsl_scripts/jenlib_test.tmpl.jdsl.groovy index 15fd94d..a2876fd 100644 --- a/e2e/jenkins-master/jen_dsl_scripts/jenlib_test.tmpl.jdsl.groovy +++ b/dockers/jenkins-master/jen_dsl_scripts/jenlib_test.tmpl.jdsl.groovy @@ -1,6 +1,6 @@ // def jobname_ci = '{{.JOB_NAME}}' // jenlib_pipe_master_ci // def github_p = '{{.GITHUB_PATH}}' // "gzvulon/jenlib" -// def jengroovy_p = '{{.JENKINS_FILE_PATH}}' // "e2e/data4test/parallel_tasks_jen/build.groovy" +// def jengroovy_p = '{{.JENKINS_FILE_PATH}}' // "data4test/parallel_tasks_jen/build.groovy" pipelineJob("${jobname_ci}") { description() diff --git a/e2e/jenkins-master/jen_dsl_scripts/new_project.jdsl.groovy b/dockers/jenkins-master/jen_dsl_scripts/new_project.jdsl.groovy similarity index 100% rename from e2e/jenkins-master/jen_dsl_scripts/new_project.jdsl.groovy rename to dockers/jenkins-master/jen_dsl_scripts/new_project.jdsl.groovy diff --git a/e2e/jenkins-master/jen_pipes/node.with.timeout.pipe.groovy b/dockers/jenkins-master/jen_pipes/node.with.timeout.pipe.groovy similarity index 100% rename from e2e/jenkins-master/jen_pipes/node.with.timeout.pipe.groovy rename to dockers/jenkins-master/jen_pipes/node.with.timeout.pipe.groovy diff --git a/e2e/jenkins-master/jen_pipes/params.pipe.groovy b/dockers/jenkins-master/jen_pipes/params.pipe.groovy similarity index 100% rename from e2e/jenkins-master/jen_pipes/params.pipe.groovy rename to dockers/jenkins-master/jen_pipes/params.pipe.groovy diff --git a/dockers/jenkins-master/jenkins.yaml b/dockers/jenkins-master/jenkins.yaml new file mode 100644 index 0000000..bc1a646 --- /dev/null +++ b/dockers/jenkins-master/jenkins.yaml @@ -0,0 +1,139 @@ +jenkins: + agentProtocols: + - "JNLP4-connect" + - "Ping" + crumbIssuer: + standard: + excludeClientIPFromCrumb: true + disableRememberMe: false + labelAtoms: + - name: "master" + markupFormatter: "plainText" + mode: NORMAL + myViewsTabBar: "standard" + numExecutors: 2 + primaryView: + all: + name: "all" + projectNamingStrategy: "standard" + quietPeriod: 5 + remotingSecurity: + enabled: false + scmCheckoutRetryCount: 0 + securityRealm: + local: + allowsSignup: true + enableCaptcha: false + users: + - id: "master" + name: "master" + properties: + - "apiToken" + - favoriting: + autofavoriteEnabled: true + - "favorite" + - "myView" + - preferredProvider: + providerId: "default" + - "timezone" + - mailer: + emailAddress: "master@master.master" + slaveAgentPort: 50000 + updateCenter: + sites: + - id: "default" + url: "https://updates.jenkins.io/update-center.json" + views: + - all: + name: "all" + viewsTabBar: "standard" +security: + apiToken: + creationOfLegacyTokenEnabled: true + tokenGenerationOnCreationEnabled: true + usageStatisticsEnabled: true + envInject: + enableLoadingFromMaster: false + enablePermissions: false + hideInjectedVars: false + globalJobDslSecurityConfiguration: + useScriptSecurity: false + sSHD: + port: -1 +unclassified: + artifactoryBuilder: + jfrogPipelinesServer: + bypassProxy: false + connectionRetries: 3 + credentialsConfig: + overridingCredentials: false + username: "****" + timeout: 300 + useCredentialsPlugin: false + bitbucketEndpointConfiguration: + endpoints: + - bitbucketCloudEndpoint: + enableCache: false + manageHooks: false + repositoriesCacheDuration: 0 + teamCacheDuration: 0 + buildDiscarders: + configuredBuildDiscarders: + - "jobBuildDiscarder" + buildTimestamp: + enableBuildTimestamp: true + pattern: "yyyy-MM-dd HH:mm:ss z" + timezone: "Etc/UTC" + fingerprints: + fingerprintCleanupDisabled: false + storage: "file" + gitHubConfiguration: + apiRateLimitChecker: ThrottleForNormalize + gitHubPluginConfig: + hookUrl: "http://localhost:8080/github-webhook/" + gitLabConnectionConfig: + connections: + - clientBuilderId: "autodetect" + connectionTimeout: 10 + ignoreCertificateErrors: false + readTimeout: 10 + gitLabServers: + servers: + - name: "default" + serverUrl: "https://gitlab.com" + gitSCM: + addGitTagAction: false + allowSecondFetch: false + createAccountBasedOnEmail: false + disableGitToolChooser: false + hideCredentials: false + showEntireCommitSummaryInChanges: false + useExistingAccountWithSameEmail: false + ivyBuildTrigger: + extendedVersionMatching: false + junitTestResultStorage: + storage: "file" + location: + adminAddress: "address not configured yet " + mailer: + charset: "UTF-8" + useSsl: false + useTls: false + mavenModuleSet: + localRepository: "default" + pluginImpl: + enableCredentialsFromNode: false + pollSCM: + pollingThreadCount: 10 + timestamper: + allPipelines: false + elapsedTimeFormat: "''HH:mm:ss.S' '" + systemTimeFormat: "''HH:mm:ss' '" +tool: + git: + installations: + - home: "git" + name: "Default" + mavenGlobalConfig: + globalSettingsProvider: "standard" + settingsProvider: "standard" diff --git a/dockers/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy b/dockers/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy new file mode 100644 index 0000000..ea414a5 --- /dev/null +++ b/dockers/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy @@ -0,0 +1,61 @@ +job("_new_ci") { + description() + keepDependencies(false) + parameters { + stringParam("seed_job_name", "_seed_job", "seed_job_name") + stringParam("seed_job_path", "file:///repo", "seed_job_path") + stringParam("seed_job_pipe", "data4test/gitlab_seeds/jobs-registrator.pipe.groovy", "seed_job_pipe") + stringParam("seed_job_branch", "master", "seed_job_branch") + stringParam("jobs_def_dir", "data4test/gitlab_seeds/jobs-defs", "jobs_def_dir") + } + disabled(false) + concurrentBuild(false) + steps { + dsl { + text("""// def seed_job_name = '{{.JOB_NAME}}' // jenlib_pipe_master_ci +// def seed_job_path = '{{.seed_job_pathATH}}' // "file:///repo" +// def seed_job_pipe = '{{.JENKINS_FILE_PATH}}' // "data4test/gitlab_seeds/jobs-registrator.pipe.groovy" +// https://code-maven.com/jenkins-pipeline-environment-variables +// https://www.iditect.com/how-to/51308400.html + +pipelineJob("\${seed_job_name}") { + description() + keepDependencies(false) + definition { + cpsScm { + scm { + git { + remote { + url("\${seed_job_path}") + } + branch("*/\${seed_job_branch}") + } + } + scriptPath("\${seed_job_pipe}") + } + } + environmentVariables { + envs(jobs_def_dir: "\${jobs_def_dir}") + keepBuildVariables(true) + } + disabled(false) + configure { + it / 'properties' / 'com.sonyericsson.rebuild.RebuildSettings' { + 'autoRebuild'('false') + 'rebuildDisabled'('false') + } + } +}""") + ignoreExisting(false) + removeAction("IGNORE") + removeViewAction("IGNORE") + lookupStrategy("JENKINS_ROOT") + } + } + configure { + it / 'properties' / 'com.sonyericsson.rebuild.RebuildSettings' { + 'autoRebuild'('false') + 'rebuildDisabled'('false') + } + } +} diff --git a/e2e/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy.yaml b/dockers/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy.yaml similarity index 87% rename from e2e/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy.yaml rename to dockers/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy.yaml index 2b90008..ee4ef6d 100644 --- a/e2e/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy.yaml +++ b/dockers/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy.yaml @@ -5,7 +5,7 @@ definition: parameters: - stringParam: ["jobname_ci", "jenlib_pipe_master_ci_X", "jenlib_pipe_master_ci"] - stringParam: ["github_p", "gzvulon/jenlib", "github_p"] - - stringParam: ["jengroovy_p", "e2e/data4test/parallel_tasks_jen/build.groovy", "jengroovy_p"] + - stringParam: ["jengroovy_p", "data4test/parallel_tasks_jen/build.groovy", "jengroovy_p"] disabled: false concurrentBuild: false steps: diff --git a/e2e/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy.yaml.short.yaml b/dockers/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy.yaml.short.yaml similarity index 87% rename from e2e/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy.yaml.short.yaml rename to dockers/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy.yaml.short.yaml index af85b90..2b5d901 100644 --- a/e2e/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy.yaml.short.yaml +++ b/dockers/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy.yaml.short.yaml @@ -5,7 +5,7 @@ definition: parameters: - stringParam: ["jobname_ci", "jenlib_pipe_master_ci_X", "jenlib_pipe_master_ci"] - stringParam: ["github_p", "gzvulon/jenlib", "github_p"] - - stringParam: ["jengroovy_p", "e2e/data4test/parallel_tasks_jen/build.groovy", "jengroovy_p"] + - stringParam: ["jengroovy_p", "data4test/parallel_tasks_jen/build.groovy", "jengroovy_p"] disabled: false concurrentBuild: false steps: diff --git a/e2e/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy.yaml.yaml b/dockers/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy.yaml.yaml similarity index 85% rename from e2e/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy.yaml.yaml rename to dockers/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy.yaml.yaml index 2e6d226..e7ce2e6 100644 --- a/e2e/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy.yaml.yaml +++ b/dockers/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy.yaml.yaml @@ -5,7 +5,7 @@ definition: | parameters { stringParam("jobname_ci", "jenlib_pipe_master_ci_X", "jenlib_pipe_master_ci") stringParam("github_p", "gzvulon/jenlib", "github_p") - stringParam("jengroovy_p", "e2e/data4test/parallel_tasks_jen/build.groovy", "jengroovy_p") + stringParam("jengroovy_p", "data4test/parallel_tasks_jen/build.groovy", "jengroovy_p") } disabled(false) concurrentBuild(false) @@ -28,7 +28,7 @@ definition: | implementation: | // def jobname_ci = '{{.JOB_NAME}}' // jenlib_pipe_master_ci // def github_p = '{{.GITHUB_PATH}}' // "gzvulon/jenlib" - // def jengroovy_p = '{{.JENKINS_FILE_PATH}}' // "e2e/data4test/parallel_tasks_jen/build.groovy" + // def jengroovy_p = '{{.JENKINS_FILE_PATH}}' // "data4test/parallel_tasks_jen/build.groovy" pipelineJob("${jobname_ci}") { description() diff --git a/e2e/jenkins-master/jobs_sys/_new_project/new_project.jdsl.groovy b/dockers/jenkins-master/jobs_sys/_new_project/new_project.jdsl.groovy similarity index 100% rename from e2e/jenkins-master/jobs_sys/_new_project/new_project.jdsl.groovy rename to dockers/jenkins-master/jobs_sys/_new_project/new_project.jdsl.groovy diff --git a/e2e/jenkins-master/jobs_sys/_new_project/new_project.pipe.groovy b/dockers/jenkins-master/jobs_sys/_new_project/new_project.pipe.groovy similarity index 100% rename from e2e/jenkins-master/jobs_sys/_new_project/new_project.pipe.groovy rename to dockers/jenkins-master/jobs_sys/_new_project/new_project.pipe.groovy diff --git a/dockers/jenkins-master/k8s/Taskfile.yml b/dockers/jenkins-master/k8s/Taskfile.yml new file mode 100644 index 0000000..12a0f77 --- /dev/null +++ b/dockers/jenkins-master/k8s/Taskfile.yml @@ -0,0 +1,38 @@ +# https://taskfile.dev + +version: '3' + +vars: + GREETING: Hello, World! + +tasks: + default: + cmds: + - echo "{{.GREETING}}" + silent: true + + ensure:ns: + desc: _ + cmds: + - kubectl create namespace jenkins + status: + - kubectl get namespaces | cut -d ' ' -f1 | grep -e '^jenkins$' + + ensure:deployment: + desc: _ + cmds: + - kubectl create -f jenkins.yaml --namespace jenkins + status: + - kubectl get pods -n jenkins | grep 'jenkins-' + + rm:depl: + desc: _ + cmds: + - kubectl delete -f jenkins.yaml --namespace jenkins + + ensure:service: + desc: _ + cmds: + - kubectl create -f jenkins-service.yaml --namespace jenkins + status: + - kubectl get services --namespace jenkins | grep 'jenkins' diff --git a/dockers/jenkins-master/k8s/jenkins-service.yaml b/dockers/jenkins-master/k8s/jenkins-service.yaml new file mode 100644 index 0000000..029eb85 --- /dev/null +++ b/dockers/jenkins-master/k8s/jenkins-service.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Service +metadata: + name: jenkins +spec: + type: NodePort + ports: + - port: 8080 + targetPort: 8080 + nodePort: 8080 + selector: + app: jenkins + +--- + +apiVersion: v1 +kind: Service +metadata: + name: jenkins-jnlp +spec: + type: ClusterIP + ports: + - port: 50000 + targetPort: 50000 + selector: + app: jenkins diff --git a/dockers/jenkins-master/k8s/jenkins.yaml b/dockers/jenkins-master/k8s/jenkins.yaml new file mode 100644 index 0000000..505baa6 --- /dev/null +++ b/dockers/jenkins-master/k8s/jenkins.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: jenkins +spec: + replicas: 1 + selector: + matchLabels: + app: jenkins + template: + metadata: + labels: + app: jenkins + spec: + containers: + - name: jenkins + image: jenconda + ports: + - name: http-port + containerPort: 8080 + - name: jnlp-port + containerPort: 50000 + # volumeMounts: + # - name: jenkins-vol + # mountPath: /var/jenkins_vol + # volumes: + # - name: jenkins-vol + # emptyDir: {} diff --git a/dockers/jenkins-master/plugins.txt b/dockers/jenkins-master/plugins.txt new file mode 100644 index 0000000..8f3241a --- /dev/null +++ b/dockers/jenkins-master/plugins.txt @@ -0,0 +1,105 @@ +ace-editor +ansible +ant +apache-httpcomponents-client-4-api +artifactory +authentication-tokens +aws-credentials +aws-java-sdk +blueocean +branch-api +build-name-setter +build-timestamp +cloudbees-folder +convert-to-pipeline +credentials +custom-build-properties +declarative-pipeline-migration-assistant-api +declarative-pipeline-migration-assistant +display-url-api +docker-commons +docker-java-api +docker-plugin +docker-workflow +durable-task +external-monitor-job +favorite +git-client +git-server +git +github +gitlab-plugin +gradle +handlebars +handy-uri-templates-2-api +htmlpublisher +ivy +jackson2-api +jira-steps +jira +jquery-detached +jsch +junit +kubernetes-pipeline-devops-steps +ldap +lockable-resources +mailer +mapdb-api +matrix-auth +matrix-project +mercurial +metrics +momentjs +multibranch-action-triggers +multibranch-build-strategy-extension +pam-auth +pipeline-as-yaml +pipeline-aws +pipeline-cps-http +pipeline-dependency-walker +pipeline-github-lib +pipeline-githubnotify-step +pipeline-graph-analysis +pipeline-milestone-step +pipeline-model-api +pipeline-rest-api +pipeline-restful-api +pipeline-stage-view +pipeline-timeline +pipeline-utility-steps +plain-credentials +pubsub-light +run-condition +scm-api +script-security +simple-build-for-pipeline +snakeyaml-api +ssh-credentials +ssh-slaves +ssh-steps +structs +subversion +timestamper +token-macro +translation +variant +windows-slaves +workflow-aggregator +workflow-api +workflow-basic-steps +workflow-cps-global-lib-http +workflow-cps-global-lib +workflow-cps +workflow-job +workflow-multibranch +workflow-scm-step +job-dsl +xml-job-to-job-dsl +gitlab-branch-source +jobConfigHistory +configuration-as-code +view-job-filters +configuration-as-code-groovy +pipeline-config-history +rebuild +envinject diff --git a/dockers/jenkins-master/plugins_manual.txt b/dockers/jenkins-master/plugins_manual.txt new file mode 100644 index 0000000..deb6254 --- /dev/null +++ b/dockers/jenkins-master/plugins_manual.txt @@ -0,0 +1,2 @@ +job-dsl +xml-job-to-job-dsl \ No newline at end of file diff --git a/dockers/jenkins-master/plugins_subdeps.txt b/dockers/jenkins-master/plugins_subdeps.txt new file mode 100644 index 0000000..cea8ee2 --- /dev/null +++ b/dockers/jenkins-master/plugins_subdeps.txt @@ -0,0 +1,32 @@ +maven-plugin:3.8 +workflow-step-api:2.22 +config-file-provider:3.6.3 +javadoc:1.5 + +credentials-binding:1.23 +cloudbees-bitbucket-branch-source:2.8.0 +blueocean-pipeline-scm-api:1.23.2 + +blueocean-bitbucket-pipeline:1.23.2 +blueocean-rest-impl:1.23.2 +blueocean-commons:1.23.2 +blueocean-jwt:1.23.2 +jenkins-design-language:1.23.2 +pipeline-build-step:2.12 +sse-gateway:1.23 +github-api:1.111 +github-branch-source:2.7.1 + +workflow-durable-task-step:2.35 +workflow-support:3.4 + + +pipeline-input-step:2.11 +pipeline-stage-step:2.3 + +bouncycastle-api +jdk-tool +trilead-api +command-launcher + + diff --git a/e2e/jenkins-master/wait-for-it.sh b/dockers/jenkins-master/wait-for-it.sh similarity index 100% rename from e2e/jenkins-master/wait-for-it.sh rename to dockers/jenkins-master/wait-for-it.sh diff --git a/dockers/jenkins-worker/Dockerfile b/dockers/jenkins-worker/Dockerfile new file mode 100755 index 0000000..2d930b2 --- /dev/null +++ b/dockers/jenkins-worker/Dockerfile @@ -0,0 +1,30 @@ +FROM continuumio/conda-ci-linux-64-python3.7 + +USER root + +ENV JENKINS_USER test_user +ENV JENKINS_PASS test_user + +USER root +RUN curl -sL https://taskfile.dev/install.sh | sh +RUN chmod +x ./bin/task && mv ./bin/task /usr/bin + +RUN apt-get update \ + && apt-get install -qqy apt-transport-https ca-certificates curl gnupg2 software-properties-common +RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - +RUN add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/debian \ + $(lsb_release -cs) \ + stable" +RUN apt-get update -qq \ + && apt-get install docker-ce -y +RUN usermod -aG docker jenkins +RUN apt-get clean +RUN curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose && chmod +x /usr/local/bin/docker-compose + +RUN /opt/conda/bin/conda install -c anaconda openjdk -y + +RUN - mkdir -p /tmp + - cd /tmp && wget https://repo.jenkins-ci.org/releases/org/jenkins-ci/plugins/swarm-client/3.23/swarm-client-3.23.jar + - mkdir -p /data/tools/ + - mv /tmp/swarm-client-3.23.jar /data/tools/ \ No newline at end of file diff --git a/dockers/jenkins-worker/Taskfile.yml b/dockers/jenkins-worker/Taskfile.yml new file mode 100644 index 0000000..43e2570 --- /dev/null +++ b/dockers/jenkins-worker/Taskfile.yml @@ -0,0 +1,32 @@ +# https://taskfile.dev + +version: '3' + +vars: + GREETING: Hello, World! + +tasks: + default: + cmds: + - echo "{{.GREETING}}" + - task -l + silent: true + + get-swarm-agent: + desc: _ + cmds: + - mkdir -p /tmp + - cd /tmp && wget https://repo.jenkins-ci.org/releases/org/jenkins-ci/plugins/swarm-client/3.23/swarm-client-3.23.jar + - mkdir -p /data/tools/ + - mv /tmp/swarm-client-3.23.jar /data/tools/ + + run-jenw-on-host: + desc: running jenkins on host + cmds: + - | + java -jar /data/tools/swarm-client-3.23.jar \ + -master http://blade2:8080 \ + -labels 'sw-agent' \ + -fsroot '.' \ + -username jenworker \ + -password jenpassword diff --git a/docs/dev.md b/docs/dev.md new file mode 100644 index 0000000..08de6a3 --- /dev/null +++ b/docs/dev.md @@ -0,0 +1,16 @@ +# Dev Information + +## Misc + +### Shared libs + +- + +```bash +â–¶ cat ../spec-file.txt | grep -v '#' | grep -v '@' | sed -e 's/http....//g' | xargs -I{} bash -c 'mkdir -p $(dirname {}) && wget http://{} -o {}' +``` \ No newline at end of file diff --git a/docs/diagrams/jenlib_configs.draw.excalidraw b/docs/diagrams/jenlib_configs.draw.excalidraw new file mode 100644 index 0000000..2443ce3 --- /dev/null +++ b/docs/diagrams/jenlib_configs.draw.excalidraw @@ -0,0 +1,1078 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://excalidraw.com", + "elements": [ + { + "type": "rectangle", + "version": 534, + "versionNonce": 92272961, + "isDeleted": false, + "id": "30QjxX7lRWtpUwBJKN5JF", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 387.6363830566406, + "y": 381.18182373046875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 208.72744750976568, + "height": 99.6363983154297, + "seed": 1768073871, + "groupIds": [ + "916GzIIolO-3K2JzOe5es" + ], + "strokeSharpness": "sharp", + "boundElementIds": [] + }, + { + "type": "text", + "version": 234, + "versionNonce": 227014479, + "isDeleted": false, + "id": "f1GXQ27XsqpZ70Za6qzGt", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 413.27293395996094, + "y": 407.86363983154297, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 172, + "height": 31, + "seed": 1905887407, + "groupIds": [ + "916GzIIolO-3K2JzOe5es" + ], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 20, + "fontFamily": 1, + "text": "src:bundle_parent", + "baseline": 23, + "textAlign": "center", + "verticalAlign": "top" + }, + { + "type": "rectangle", + "version": 104, + "versionNonce": 1985595558, + "isDeleted": false, + "id": "whHmtz92RheApegkJIzEc", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 670.9091796875, + "y": 387.15911865234375, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 216.72729492187503, + "height": 100.36358642578126, + "seed": 1702193647, + "groupIds": [ + "6FlHQ53rVyvl5DzF3bZE6" + ], + "strokeSharpness": "sharp", + "boundElementIds": [ + "aFBwWTGMLibQzCeq-KJb6" + ] + }, + { + "type": "text", + "version": 58, + "versionNonce": 392127846, + "isDeleted": false, + "id": "XRmt9-xs6gvB7VrSP-RDr", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 709.45458984375, + "y": 404.022705078125, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 153, + "height": 31, + "seed": 1462451311, + "groupIds": [ + "6FlHQ53rVyvl5DzF3bZE6" + ], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 20, + "fontFamily": 1, + "text": "generated_cfgs", + "baseline": 23, + "textAlign": "left", + "verticalAlign": "top" + }, + { + "type": "ellipse", + "version": 491, + "versionNonce": 1437687041, + "isDeleted": false, + "id": "TAbSx26c1yHkP-j0Q0ot0", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 697.4545288085938, + "y": 245.90907287597656, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 106.90911865234375, + "height": 98.1817626953125, + "seed": 46408751, + "groupIds": [ + "g4uK4aPRlEal1BND0uQQS" + ], + "strokeSharpness": "sharp", + "boundElementIds": [] + }, + { + "type": "text", + "version": 36, + "versionNonce": 509587343, + "isDeleted": false, + "id": "8cPFIDATThpLcGJQzir33", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 712.9090881347656, + "y": 279.4999542236328, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 76, + "height": 31, + "seed": 110707631, + "groupIds": [ + "g4uK4aPRlEal1BND0uQQS" + ], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 20, + "fontFamily": 1, + "text": "gen:cfgs", + "baseline": 23, + "textAlign": "center", + "verticalAlign": "top" + }, + { + "type": "ellipse", + "version": 559, + "versionNonce": 1012929761, + "isDeleted": false, + "id": "T1sh_K_lxFKl6UdOQy2l2", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 441.0000305175781, + "y": 243.63641357421875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 106.90911865234375, + "height": 98.1817626953125, + "seed": 339776527, + "groupIds": [ + "FGl_IvNqCkksjlkGf8AFL" + ], + "strokeSharpness": "sharp", + "boundElementIds": [] + }, + { + "type": "text", + "version": 135, + "versionNonce": 1224795567, + "isDeleted": false, + "id": "ujuNulPf4ERmz2QyHoUyS", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 450.95458984375, + "y": 277.227294921875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 87, + "height": 31, + "seed": 1020115041, + "groupIds": [ + "FGl_IvNqCkksjlkGf8AFL" + ], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 20, + "fontFamily": 1, + "text": "fetch:src", + "baseline": 23, + "textAlign": "center", + "verticalAlign": "top" + }, + { + "type": "arrow", + "version": 54, + "versionNonce": 1881985217, + "isDeleted": false, + "id": "i2FXJ-Vl60_dslaKCL8Jz", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 498.18182373046875, + "y": 346.2727355957031, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 2.18182373046875, + "height": 34.909088134765625, + "seed": 1596382991, + "groupIds": [], + "strokeSharpness": "round", + "boundElementIds": [], + "points": [ + [ + 0, + 0 + ], + [ + -2.18182373046875, + 34.909088134765625 + ] + ], + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "type": "arrow", + "version": 32, + "versionNonce": 482380794, + "isDeleted": false, + "id": "aFBwWTGMLibQzCeq-KJb6", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 754.9091186523438, + "y": 344.0909118652344, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 0.727294921875, + "height": 34.909088134765625, + "seed": 1880595183, + "groupIds": [], + "strokeSharpness": "round", + "boundElementIds": [], + "endBinding": { + "elementId": "whHmtz92RheApegkJIzEc", + "focus": -0.24044131815397937, + "gap": 8.15911865234375 + }, + "points": [ + [ + 0, + 0 + ], + [ + -0.727294921875, + 34.909088134765625 + ] + ], + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "type": "arrow", + "version": 19, + "versionNonce": 900617345, + "isDeleted": false, + "id": "_JzJwyynX88EGfu1Wlczk", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 770.9091186523438, + "y": 540.6363830566406, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 146.18182373046875, + "height": 4.36358642578125, + "seed": 1756500015, + "groupIds": [], + "strokeSharpness": "round", + "boundElementIds": [], + "points": [ + [ + 0, + 0 + ], + [ + 146.18182373046875, + -4.36358642578125 + ] + ], + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "type": "diamond", + "version": 30, + "versionNonce": 282205281, + "isDeleted": false, + "id": "oe5Udf3IdgxyTST1OIYZb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 973.0909423828125, + "y": 518.0909118652344, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 244.3636474609375, + "height": 74.90908813476562, + "seed": 1431253697, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [] + }, + { + "type": "text", + "version": 22, + "versionNonce": 842339887, + "isDeleted": false, + "id": "SZZVArO39ECNjTaWUHp7d", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1021.8182983398438, + "y": 541.5, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 127, + "height": 31, + "seed": 2070697185, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 20, + "fontFamily": 1, + "text": "pre build info", + "baseline": 23, + "textAlign": "left", + "verticalAlign": "top" + }, + { + "type": "arrow", + "version": 33, + "versionNonce": 133294145, + "isDeleted": false, + "id": "kh4UHpUpekaPPkaWN0OcG", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 776.727294921875, + "y": 655.5454711914062, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 145.45452880859375, + "height": 5.81817626953125, + "seed": 1800939247, + "groupIds": [], + "strokeSharpness": "round", + "boundElementIds": [], + "points": [ + [ + 0, + 0 + ], + [ + 145.45452880859375, + -5.81817626953125 + ] + ], + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "type": "diamond", + "version": 42, + "versionNonce": 1156397089, + "isDeleted": false, + "id": "A3_NtSJIWmIflNSb4YTtI", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 965.0909423828125, + "y": 634.45458984375, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 283.6363525390625, + "height": 76.36358642578125, + "seed": 784661295, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [] + }, + { + "type": "text", + "version": 23, + "versionNonce": 901420655, + "isDeleted": false, + "id": "tK65CI2SiIsyHXv74LUDF", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1031.272705078125, + "y": 656.4091796875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 140, + "height": 31, + "seed": 1046093775, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 20, + "fontFamily": 1, + "text": "post build info", + "baseline": 23, + "textAlign": "left", + "verticalAlign": "top" + }, + { + "type": "diamond", + "version": 128, + "versionNonce": 1172939238, + "isDeleted": false, + "id": "l6wKWQJT5MHJe7gZjHSKJ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 693.8466186523438, + "y": 557.1591491699219, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 368.72723388671886, + "height": 71.99999999999999, + "seed": 640185089, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [] + }, + { + "type": "text", + "version": 32, + "versionNonce": 776233953, + "isDeleted": false, + "id": "Umfcy5aodzdLQr6LarWY5", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 810.9091186523438, + "y": 583.6818237304688, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 181, + "height": 31, + "seed": 1813979247, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 20, + "fontFamily": 1, + "text": "gengeter tasks.yml", + "baseline": 23, + "textAlign": "left", + "verticalAlign": "top" + }, + { + "type": "text", + "version": 96, + "versionNonce": 1976845798, + "isDeleted": false, + "id": "gtUq709crZrZc4hLAIw74", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 328.0000305175781, + "y": 656.852294921875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 230, + "height": 94, + "seed": 1310721857, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 20, + "fontFamily": 1, + "text": "task ci:gen-build-plan\ntask -d build_plant run\n", + "baseline": 85, + "textAlign": "left", + "verticalAlign": "top" + }, + { + "type": "rectangle", + "version": 35, + "versionNonce": 1798334191, + "isDeleted": false, + "id": "zV_lOxGwdjytotTixgjWk", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 454.54547119140625, + "y": 785.5454711914062, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 139.6363525390625, + "height": 100.36367797851562, + "seed": 497396783, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [] + }, + { + "type": "text", + "version": 15, + "versionNonce": 1805714305, + "isDeleted": false, + "id": "u5bYfj8djMGwAr1aunS7A", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 490.18182373046875, + "y": 812.5909118652344, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 89, + "height": 31, + "seed": 1001427009, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 20, + "fontFamily": 1, + "text": "json cfgs", + "baseline": 23, + "textAlign": "left", + "verticalAlign": "top" + }, + { + "type": "rectangle", + "version": 36, + "versionNonce": 1922493281, + "isDeleted": false, + "id": "eu77Lj_1uJIOQe9EsxPjx", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 455.2727355957031, + "y": 915.727294921875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 165.09091186523438, + "height": 91.6363525390625, + "seed": 902221679, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [] + }, + { + "type": "text", + "version": 13, + "versionNonce": 1462286127, + "isDeleted": false, + "id": "kysdRL6Po_YbT5fDbdNZX", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 485.8182373046875, + "y": 960.95458984375, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 66, + "height": 31, + "seed": 434043105, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 20, + "fontFamily": 1, + "text": "scripts", + "baseline": 23, + "textAlign": "left", + "verticalAlign": "top" + }, + { + "type": "ellipse", + "version": 84, + "versionNonce": 373610305, + "isDeleted": false, + "id": "w6uA2LEQrRc_rATcyhKcr", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 670.5455322265625, + "y": 831.3637084960938, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 152, + "height": 130.90908813476562, + "seed": 384900129, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [] + }, + { + "type": "arrow", + "version": 26, + "versionNonce": 2122840399, + "isDeleted": false, + "id": "1Q1NGOc1AJ4KqJVJW3H9k", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 595.6363525390625, + "y": 830.6363830566406, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 79.2728271484375, + "height": 36.363616943359375, + "seed": 1223676641, + "groupIds": [], + "strokeSharpness": "round", + "boundElementIds": [], + "points": [ + [ + 0, + 0 + ], + [ + 79.2728271484375, + 36.363616943359375 + ] + ], + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "type": "arrow", + "version": 44, + "versionNonce": 151668513, + "isDeleted": false, + "id": "mS-bw8NGuEmzR_8NypDlR", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 621.8181762695312, + "y": 969.5454711914062, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 63.2728271484375, + "height": 22.54547119140625, + "seed": 1520687951, + "groupIds": [], + "strokeSharpness": "round", + "boundElementIds": [], + "points": [ + [ + 0, + 0 + ], + [ + 63.2728271484375, + -22.54547119140625 + ] + ], + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "type": "arrow", + "version": 30, + "versionNonce": 89570049, + "isDeleted": false, + "id": "3Rs5zkQCWRWU6iCRPBSEJ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 821.8181762695312, + "y": 896.8182067871094, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 102.54547119140625, + "height": 108.3636474609375, + "seed": 587782081, + "groupIds": [], + "strokeSharpness": "round", + "boundElementIds": [], + "points": [ + [ + 0, + 0 + ], + [ + 102.54547119140625, + -108.3636474609375 + ] + ], + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "type": "diamond", + "version": 32, + "versionNonce": 1269412577, + "isDeleted": false, + "id": "PVD6giSlcwGfwxmHh1hAr", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 949.8182983398438, + "y": 729.5454711914062, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 238.54534912109375, + "height": 133.81817626953125, + "seed": 1749282913, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [] + }, + { + "type": "text", + "version": 29, + "versionNonce": 1663575983, + "isDeleted": false, + "id": "0PznzXkr0EtqY1LlY-22K", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1005.8182983398438, + "y": 773.3182373046875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 228, + "height": 31, + "seed": 909205007, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 20, + "fontFamily": 1, + "text": "groovy file with stages", + "baseline": 23, + "textAlign": "left", + "verticalAlign": "top" + }, + { + "type": "text", + "version": 14, + "versionNonce": 1620225729, + "isDeleted": false, + "id": "e3EVF-ZrQF0Odd9_IWrm4", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 703.2727661132812, + "y": 809.6818237304688, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 63, + "height": 31, + "seed": 760104911, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 20, + "fontFamily": 1, + "text": "groovy", + "baseline": 23, + "textAlign": "left", + "verticalAlign": "top" + }, + { + "type": "text", + "version": 13, + "versionNonce": 709188047, + "isDeleted": false, + "id": "TejVtdnL3I2WMBrwxYtDW", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 720.727294921875, + "y": 962.4091796875, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 61, + "height": 31, + "seed": 477590497, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 20, + "fontFamily": 1, + "text": "python", + "baseline": 23, + "textAlign": "left", + "verticalAlign": "top" + }, + { + "type": "arrow", + "version": 35, + "versionNonce": 179383969, + "isDeleted": false, + "id": "O9H6uGIjbaOUAFpIovJKU", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 791.2727661132812, + "y": 981.1818237304688, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 158.5455322265625, + "height": 22.54547119140625, + "seed": 857205071, + "groupIds": [], + "strokeSharpness": "round", + "boundElementIds": [], + "points": [ + [ + 0, + 0 + ], + [ + 158.5455322265625, + 22.54547119140625 + ] + ], + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "type": "diamond", + "version": 33, + "versionNonce": 280277633, + "isDeleted": false, + "id": "WpShEhSsQLoILPjSz2tIG", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 991.2727661132812, + "y": 939, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 270.54559326171875, + "height": 125.09100341796875, + "seed": 558021615, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [] + }, + { + "type": "arrow", + "version": 13, + "versionNonce": 134640143, + "isDeleted": false, + "id": "nx_93tyTsnR1kBtwsPHj2", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1125.0909423828125, + "y": 938.2727661132812, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 41.45458984375, + "height": 67.63638305664062, + "seed": 836074991, + "groupIds": [], + "strokeSharpness": "round", + "boundElementIds": [], + "points": [ + [ + 0, + 0 + ], + [ + -41.45458984375, + -67.63638305664062 + ] + ], + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow" + }, + { + "type": "text", + "version": 15, + "versionNonce": 95115311, + "isDeleted": false, + "id": "jcGMMFCJyGjJnXiocJCZa", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1125.0909423828125, + "y": 881.6818237304688, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 47, + "height": 63, + "seed": 78402511, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 20, + "fontFamily": 1, + "text": "jenlib\n", + "baseline": 54, + "textAlign": "left", + "verticalAlign": "top" + }, + { + "type": "text", + "version": 32, + "versionNonce": 2138621505, + "isDeleted": false, + "id": "Ch4ZsI2-kElK797mBIw2F", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1072.727294921875, + "y": 982.7727661132812, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 227, + "height": 31, + "seed": 464945839, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElementIds": [], + "fontSize": 20, + "fontFamily": 1, + "text": "yaml file configurations", + "baseline": 23, + "textAlign": "left", + "verticalAlign": "top" + } + ], + "appState": { + "viewBackgroundColor": "#ffffff", + "gridSize": null, + "scrollX": 0, + "scrollY": -586.6666870117188, + "zoom": { + "value": 0.5, + "translation": { + "x": 171.66666666666634, + "y": 232.99999999999994 + } + } + } +} \ No newline at end of file diff --git a/docs/groovy.md b/docs/groovy.md new file mode 100644 index 0000000..81c0ce3 --- /dev/null +++ b/docs/groovy.md @@ -0,0 +1,37 @@ +# Groovy Essentials + +## Basic + +### Variables + +### Methods + +## Difference from Python + +### kwargs and args + +### method call + +## Imports and Dymanic loading + +### Load Class (import) from custom groovy file + +> url_from= + +```groovy +File sourceFile = new File("/full/path/to/./src/org/jenslib/jentools.groovy"); +Class groovyClass = new GroovyClassLoader(getClass().getClassLoader()).parseClass(sourceFile); +GroovyObject myObject = (GroovyObject) groovyClass.newInstance(); +``` + +### Evaluate code ad-hoc + +> Evaluate text url_from= + +```groovy +// evaluate implicitly creates a class based on the filename specified +evaluate(new File("./Testutils.groovy")) +// Safer to use 'def' here as Groovy seems fussy about whether the filename (and therefore implicit class name) has a capital first letter +def tu = new Testutils() +tu.myUtilityMethod("hello world") +``` diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..1904e01 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,25 @@ +# Jenlib. Jenkins Automation Library + +Jenlib is Shared + +Basic Usage: see [README.md](./var/README.md) + +## Job DSL + +- https://github.com/Ticketfly/jenkins-docker-examples +- https://www.praqma.com/stories/dockerized-jenkins-jobdsl/ +- https://github.com/thomasleveil/docker-jenkins-dsl-ready +- https://github.com/jenkinsci/job-dsl-plugin/wiki/User-Power-Moves#run-a-dsl-script-locally +- https://www.jenkins.io/blog/2019/08/23/introducing-gitlab-branch-source-plugin/ +- https://gist.github.com/baymac/f1a2249a0ec7b999c057056937e752a6 + + +## References + +- [Jenkins Shared Libraries](https://www.jenkins.io/doc/book/pipeline/shared-libraries/) +- [Change Workspace Root](https://stackoverflow.com/questions/34854377/how-to-change-workspace-and-build-record-root-directory-on-jenkins) +- [Using ws](https://issues.jenkins.io/browse/JENKINS-41446) +- [Python client](https://python-jenkins.readthedocs.io/en/latest/examples.html#example-1-get-version-of-jenkins) +- [Material Docker](https://hub.docker.com/r/squidfunk/mkdocs-material/) +- +- \ No newline at end of file diff --git a/docs/jenkins.md b/docs/jenkins.md new file mode 100644 index 0000000..82fca1f --- /dev/null +++ b/docs/jenkins.md @@ -0,0 +1,4 @@ +# Jenkins + +- +- \ No newline at end of file diff --git a/docs/limitations.md b/docs/limitations.md new file mode 100644 index 0000000..7af7702 --- /dev/null +++ b/docs/limitations.md @@ -0,0 +1,15 @@ +# Jenkins Tasks Limitations + +## Restrictions + +> ci-flow + +- supported: + - clean task call + - `task TASK_NAME` + - `task: TASK_NAME` + +- unsupported: + - no variable are allowed + - `task TASK_NAME VAR=val` + - `task: TASK_NAME, vars: VAR:val` diff --git a/docs/links.md b/docs/links.md new file mode 100644 index 0000000..2831eaa --- /dev/null +++ b/docs/links.md @@ -0,0 +1,5 @@ +# Links To Read + +- [Checkout to Local Branch MBP](https://support.cloudbees.com/hc/en-us/articles/226122247-How-to-Customize-Checkout-for-Pipeline-Multibranch-) +- [setup predefined token for user](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) +- diff --git a/docs/release_notes.md b/docs/release_notes.md new file mode 100644 index 0000000..5efc0ac --- /dev/null +++ b/docs/release_notes.md @@ -0,0 +1,9 @@ +# Jenlib Release Notes + +## jenlib-0.5.1-rc.20202312.rc + +- adopt git project version dir approach + +## jenlib-0.4.3-rc.20190101.rc + +- introduce parallel deps diff --git a/docs/s.drawio b/docs/s.drawio new file mode 100644 index 0000000..0f44747 --- /dev/null +++ b/docs/s.drawio @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/tutorials/.gitignore b/docs/tutorials/.gitignore new file mode 100644 index 0000000..9258702 --- /dev/null +++ b/docs/tutorials/.gitignore @@ -0,0 +1 @@ +tut_* \ No newline at end of file diff --git a/docs/tutorials/3000.tasker.variables/Taskfile.yml b/docs/tutorials/3000.tasker.variables/Taskfile.yml new file mode 100644 index 0000000..5c554f8 --- /dev/null +++ b/docs/tutorials/3000.tasker.variables/Taskfile.yml @@ -0,0 +1,20 @@ +# https://taskfile.dev + +version: '3' + +output: prefixed + +includes: + order: order.yml + envvar: envvar.yml + +vars: + GREETING: Task Variables + +tasks: + default: + cmds: + - echo "{{.GREETING}}" + - echo ">> task -l for task list" + silent: true + prefix: "{{.GREETING}}" diff --git a/docs/tutorials/3000.tasker.variables/envvar.yml b/docs/tutorials/3000.tasker.variables/envvar.yml new file mode 100644 index 0000000..6ee1698 --- /dev/null +++ b/docs/tutorials/3000.tasker.variables/envvar.yml @@ -0,0 +1,36 @@ +# https://taskfile.dev + +version: '3' +output: interleaved +vars: + GREETING: Task Variables + IN_FILE: "def:task:file" + _inf: "[info ]" + _err: "[error ]" + _wrn: "[warn ]" + _svr: "[severe]" + _ntc: "[notice]" + _trc: "[trace ]" + + JENTOOLS: "$(realpath tools)" + +env: + PATH: "{{ .JENTOOLS }}:{{.PATH}}" + +tasks: + default: + cmds: + - echo "{{.GREETING}}" + - echo ">> task -l for task list" + silent: true + + test-path: + desc: _ + cmds: + - venv-run which conda + + test-path2: + desc: _ + cmds: + - venv-run which conda + diff --git a/docs/tutorials/3000.tasker.variables/order.yml b/docs/tutorials/3000.tasker.variables/order.yml new file mode 100644 index 0000000..086008f --- /dev/null +++ b/docs/tutorials/3000.tasker.variables/order.yml @@ -0,0 +1,93 @@ +# https://taskfile.dev + +version: '3' +output: interleaved +vars: + GREETING: Task Variables + IN_FILE: "def:task:file" + _inf: "[info ]" + _err: "[error ]" + _wrn: "[warn ]" + _svr: "[severe]" + _ntc: "[notice]" + _trc: "[trace ]" + +tasks: + default: + cmds: + - echo "{{.GREETING}}" + - echo ">> task -l for task list" + silent: true + + _print_cmds: &ref__print_cmds + desc: "{{.TASK}}" + cmds: + - echo "{{._inf}} [{{.TASK}}] IN_FILE='{{ .IN_FILE }}'" + - echo "{{._inf}} [{{.TASK}}] IN_TASK='{{ .IN_TASK }}'" + - echo "{{._inf}} [{{.TASK}}] IN_SHELL='{{ .IN_SHELL }}'" + silent: true + + print-clean: + <<: *ref__print_cmds + desc: No var defined in task + + print-stage-def: + <<: *ref__print_cmds + vars: + IN_TASK: def:task:stage + desc: No var defined in task + + print-stage-redefine: + <<: *ref__print_cmds + desc: tasker var redefined + vars: + IN_TASK: def:task:stage + IN_FILE: def:task:stage + + assert-out: + args: + cmd: command + args: more arguments + assert: present/missing required=false + out: output + vars: + v_opt: '{{if eq .assert "missing"}}-v{{else}}{{end}}' + cmds: + - | + {{.cmd}} {{.args}} | grep {{.v_opt}} "{{.out}}" + + flow-show-vars: + desc: _ + cmds: + - task: assert-out + vars: + cmd: task order:print-stage-def + args: IN_FILE=redefined:shell + assert: present + out: IN_FILE='redefined:shell' + - task: assert-out + vars: + cmd: task order:print-stage-def + args: IN_TASK=redefined:shell + assert: missing + out: IN_TASK='redefined:shell' + + + _flow-show-vars_org: + # desc: _ + cmds: + - task order:print-stage-def IN_FILE=sdfsdf grep "IN_FILE='sdfsdf'" + - task order:print-stage-def IN_TASK=sdfsdf grep -v "IN_TASK='sdfsdf'" + + multi-print: + desc: _ + vars: + raw_items: "$(yq r order.yml tasks --tojson | jq -r 'keys[]' | grep -e '^print-')" + gen_items: "task -l | cut -d'*' -f2 | cut -d' ' -f2 | grep 'print-' | sed -e 's/:$//g'" + cmds: + - echo "{{.gen_items}}" + - "{{.gen_items}}" + - echo $({{.gen_items}}) + - echo "{{._inf}} ---" + - task --silent $({{.gen_items}}) + diff --git a/docs/tutorials/3000.tasker.variables/tools/venv-ensure b/docs/tutorials/3000.tasker.variables/tools/venv-ensure new file mode 100644 index 0000000..13008e1 --- /dev/null +++ b/docs/tutorials/3000.tasker.variables/tools/venv-ensure @@ -0,0 +1,3 @@ +#!/bin/bash + +echo $0 \ No newline at end of file diff --git a/docs/tutorials/3000.tasker.variables/tools/venv-run b/docs/tutorials/3000.tasker.variables/tools/venv-run new file mode 100755 index 0000000..13008e1 --- /dev/null +++ b/docs/tutorials/3000.tasker.variables/tools/venv-run @@ -0,0 +1,3 @@ +#!/bin/bash + +echo $0 \ No newline at end of file diff --git a/docs/tutorials/4000.jenlib.parallel/samples.md b/docs/tutorials/4000.jenlib.parallel/samples.md new file mode 100644 index 0000000..0f9157f --- /dev/null +++ b/docs/tutorials/4000.jenlib.parallel/samples.md @@ -0,0 +1,49 @@ +# samples + +## Groovy to task + +> compare:stage.get-submodules:groovy + +```groovy +stage("get-submodules") { + steps { + script { + dir("pack-dir") { + sh "git submodule update --recursive --init --jobs 16" + sh "du -sh ." + dir ("the-dir") { + if (env.GITLAB_OBJECT_KIND == 'push') { + echo "it's from push" + sh "git checkout ${BRANCH_NAME}" + // sh "git clean -fdx && git reset --hard && git pull" + } else { + echo "it's from MR" + sh "rsync -zrvh ${env.WORKSPACE}/subdir/* ." + } + } + } + } + } +} +``` + +> compare:stage.get-submodules:task.yml + +```yaml +get-submodules: + dir: pack-dir + cmds: + - git submodule update --recursive --init --jobs 16 + - du -sh . + - | + pushd the-dir + if [[ "${GITLAB_OBJECT_KIND}" == 'push' ]]; then + echo "it's from push" + git checkout ${BRANCH_NAME} + # git clean -fdx && git reset --hard && git pull + else + echo "it's from MR" + rsync -zrvh ${env.WORKSPACE}/subdir/* . + fi + popd +``` \ No newline at end of file diff --git a/docs/tutorials/reposet.conf.yml b/docs/tutorials/reposet.conf.yml new file mode 100644 index 0000000..b96a84c --- /dev/null +++ b/docs/tutorials/reposet.conf.yml @@ -0,0 +1,4 @@ +aguide: + branch: master + dirname: aguide + remote_origin_url: git@github.com:yairdar/aguide.git \ No newline at end of file diff --git a/docs/tutorials/reposet.tasks.yml b/docs/tutorials/reposet.tasks.yml new file mode 100755 index 0000000..918c111 --- /dev/null +++ b/docs/tutorials/reposet.tasks.yml @@ -0,0 +1,157 @@ + #!/usr/local/bin/task --taskfile + # https://taskfile.dev + + version: '3' + # output: prefixed + + vars: + runconf_path: reposet.conf.yml + repos: "$(echo $(yq r {{.runconf_path}} --tojson | jq -r '.|keys[]'))" + # taskpath_arg: "-t ./rgc.yml" + taskpath_arg: "" + task_opt: '-o prefixed' + + tasks: + # --initial repo creatin + init: + desc: initialize this dir with current reposet + cmds: + - task: list-git-repos + vars: + last_suffix: '| tee > "{{.runconf_path}}" ' + + list-git-repos: + desc: _ + arg: + last_suffix: default='' desc='' + vars: + path: "." + to_yaml: "| jq -s '.'| jq 'reduce .[] as $i ({}; .[$i.dirname] = $i)' | yq r - -P" + rest_choice: "{{ if .tojson }}{{else}}{{.to_yaml}}{{end}}" + cmds: + - echo {{.path}} + - | + ls -1 {{.path}} \ + | xargs -n1 -I{} bash -c \ + "test -e {{.path}}/{}/.git && echo {} || true" \ + | xargs -n1 -I{} bash -c \ + 'echo \ + - {"dirname": "{}", \ + "branch": "$(git -C {{.path}}/{} rev-parse --abbrev-ref HEAD)", \ + "remote_origin_url": "$(git -C {{.path}}/{} config --get remote.origin.url)", \ + "commit": "$(git -C {{.path}}/{} rev-parse --short HEAD)"} \ + ' | yq r - '[*]' \ + --tojson {{.rest_choice}} {{.last_suffix}} + + silent: true + + # --- multicast task implementation --- + multi-bash: + _desc: run multiple inputs in parallel + args: + cmd: command + with_items: '[cmd({}) for {} in with_items]' + jenlib: + parallel: with_items + item: '{}' + vars: + count_intems: "$(echo {{.with_items}} | wc -w)" + cmds: + - echo "count_intems={{.count_intems}}" + - | + echo "{{.with_items}}" \ + | xargs -n 1 \ + | xargs \ + -I {} \ + -P {{.count_intems}} \ + {{.cmd}} + prefix: "par:{{.count_intems}}" + silent: true + + do-many: + jenlib: {multitask: true, items: with_items, parallel: unfold, subseq: unfold} + args: + taskname: install-src:one + cmds: + - task: multi-bash + vars: + with_items: "{{.repos}}" + cmd: "task {{.task_opt}} {{.taskpath_arg}} {{.taskname}} repo='{}'" + + many-by-one: &_ref_many_by_one + desc: _ + cmds: + - task: do-many + vars: {taskname: "{{.TASK}}:one"} + silent: true + + # --- ones --- + _repo_vars:one: &_ref_one + vars: + dirname: "$(yq read {{ .runconf_path }} {{.repo}}.dirname)" + remote_origin_url: "$(yq read {{ .runconf_path }} {{.repo}}.remote_origin_url)" + branch: "$(yq read {{ .runconf_path }} {{.repo}}.branch)" + prefix: "{{.TASK}}:{{.repo}}" + silent: true + + install-src:one: + <<: *_ref_one + cmds: + - echo task -d {{.dirname}} dev:install-here + install-src: *_ref_many_by_one + + status:one: + <<: *_ref_one + cmds: + - test -e {{.dirname}} && echo "{{.repo}}=present" || echo "{{.repo}}=missing" + status: *_ref_many_by_one + + clone:one: + <<: *_ref_one + cmds: + - git clone {{.remote_origin_url}} {{.dirname}} + status: + - test -d {{.dirname}} + clone: *_ref_many_by_one + + checkout:one: + <<: *_ref_one + cmds: + - git -C {{.dirname}} checkout {{.branch}} + status: + - | + [[ "$(git -C {{.dirname}} rev-parse --abbrev-ref HEAD)" == "{{.branch}}" ]] + checkout: *_ref_many_by_one + + pull:one: + <<: *_ref_one + cmds: + - git -C {{.dirname}} pull --rebase + pull: *_ref_many_by_one + + push:one: + <<: *_ref_one + cmds: + - git -C {{.dirname}} push + push: *_ref_many_by_one + + sync:one: + <<: *_ref_one + env: + dirname: "{{.dirname}}" + cmds: + - task: pull:one + - task: push:one + sync: *_ref_many_by_one + + resolve:one: + <<: *_ref_one + env: + dirname: "{{.dirname}}" + cmds: + - task: clone:one + - task: checkout:one + - task: pull:one + resolve: *_ref_many_by_one + # # --- reposet generation --- + diff --git a/drem/layout.md b/drem/layout.md new file mode 100644 index 0000000..8dc55c1 --- /dev/null +++ b/drem/layout.md @@ -0,0 +1,26 @@ +# Data Layout + +```bash +run_id/ +clip1/0001/Wheels/ + - p1 + - p2 + + +*/*/Wheels/ + +*/*/Wheels/ + + +clip1/*/* +- unifyed struct top-level + +tech/clip/frame + +read() + + +partition per clip +ordered by frame +https://github.com/argoproj/argo-workflows/blob/master/examples/daemon-step.yaml +``` \ No newline at end of file diff --git a/drem/sam_drem.py b/drem/sam_drem.py new file mode 100644 index 0000000..e8d2541 --- /dev/null +++ b/drem/sam_drem.py @@ -0,0 +1,31 @@ +from dremio_client import init +client = init() # initialise connectivity to Dremio via config file +catalog = client.data # fetch catalog + +# vds = catalog.space.vds.get() # fetch a specific dataset +# df = vds.query() # query the first 1000 rows of the dataset and return as a DataFrame +# pds = catalog.source.pds.get() # fetch a physical dataset +# pds.metadata_refresh() # refresh metadata on that dataset +print("[INFO] im ok") + +q = """ +SELECT * FROM s3."ydmoneytime"."or_data"."fiff_db_example"."nor_json"."pext_20-02-27_10-19-41_Bella_DATACO-354_36001_47999.FEIF_Master-state.docs.framedata.jsonl" WHERE x > 0 LIMIT 10 +""" + +q = """ +'SELECT * FROM mes3."ydmoneytime"."or_data"."fiff_db_example"."nor_json"."pext_20-02-27_10-19-41_Bella_DATACO-354_36001_47999.FEIF_Wheels-meas.docs.framedata.jsonl" WHERE x > 0 LIMIT 10' +""" + +q2 =""" +SELECT T1.T0grabIdx, T1.id, T1.x, T2.p +FROM s3."ydmoneytime"."or_data"."fiff_db_example"."nor_json"."pext_20-02-27_10-19-41_Bella_DATACO-354_36001_47999.FEIF_Master-state.docs.framedata.jsonl" T1 +JOIN s3."ydmoneytime"."or_data"."fiff_db_example"."nor_json"."pext_20-02-27_10-19-41_Bella_DATACO-354_36001_47999.FEIF_Wheels-state.docs.framedata.jsonl" T2 +ON T1.T0grabIdx = T2.T0grabIdx and T1.id = T2.id +WHERE T1.x > 0 +LIMIT 10 +""" + +from pprint import pprint +res = client.query(q2) +# pprint(res) +pprint(res) \ No newline at end of file diff --git a/e2e/jenkins-master/e2e/Taskfile.yml b/e2e/jenkins-master/e2e/Taskfile.yml deleted file mode 100644 index 069f008..0000000 --- a/e2e/jenkins-master/e2e/Taskfile.yml +++ /dev/null @@ -1,29 +0,0 @@ -# https://taskfile.dev - -version: '2.3' - -vars: - GREETING: E2E tests with docker-machine - jentest_machine: {sh: cat cfg/_machine.name.txt} - -tasks: - default: - cmds: - - echo "{{.GREETING}}" - - task -l - silent: true - - dm:list: - desc: _ - cmds: - - docker-machine ls - - tpa: - desc: _ - cmds: - - curl http://192.168.99.100:8880/ - - curl http://192.168.99.100:8880/ | grep 'Please wait while Jenkins is getting ready to work' - - - - diff --git a/e2e/jenkins-master/e2e/cfg/_machine.name.txt b/e2e/jenkins-master/e2e/cfg/_machine.name.txt deleted file mode 100644 index c937191..0000000 --- a/e2e/jenkins-master/e2e/cfg/_machine.name.txt +++ /dev/null @@ -1 +0,0 @@ -jentest \ No newline at end of file diff --git a/e2e/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy b/e2e/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy deleted file mode 100644 index 2d57e86..0000000 --- a/e2e/jenkins-master/jobs_sys/_new_ci/_new_ci.jdsl.groovy +++ /dev/null @@ -1,53 +0,0 @@ -job("_new_ci") { - description() - keepDependencies(false) - parameters { - stringParam("jobname_ci", "jenlib_pipe_master_ci_X", "jenlib_pipe_master_ci") - stringParam("github_p", "gzvulon/jenlib", "github_p") - stringParam("jengroovy_p", "e2e/data4test/parallel_tasks_jen/build.groovy", "jengroovy_p") - } - disabled(false) - concurrentBuild(false) - steps { - dsl { - text("""// def jobname_ci = '{{.JOB_NAME}}' // jenlib_pipe_master_ci -// def github_p = '{{.GITHUB_PATH}}' // "gzvulon/jenlib" -// def jengroovy_p = '{{.JENKINS_FILE_PATH}}' // "e2e/data4test/parallel_tasks_jen/build.groovy" - -pipelineJob("\${jobname_ci}") { - description() - keepDependencies(false) - definition { - cpsScm { - scm { - git { - remote { - github("\${github_p}", "https") - } - branch("*/master") - } - } - scriptPath("\${jengroovy_p}") - } - } - disabled(false) - configure { - it / 'properties' / 'com.sonyericsson.rebuild.RebuildSettings' { - 'autoRebuild'('false') - 'rebuildDisabled'('false') - } - } -}""") - ignoreExisting(false) - removeAction("IGNORE") - removeViewAction("IGNORE") - lookupStrategy("JENKINS_ROOT") - } - } - configure { - it / 'properties' / 'com.sonyericsson.rebuild.RebuildSettings' { - 'autoRebuild'('false') - 'rebuildDisabled'('false') - } - } -} diff --git a/e2e/jenkins-master/plugins.txt b/e2e/jenkins-master/plugins.txt deleted file mode 100644 index a2fd609..0000000 --- a/e2e/jenkins-master/plugins.txt +++ /dev/null @@ -1,141 +0,0 @@ -ace-editor:1.1 -ansible:1.0 -ant:1.11 -apache-httpcomponents-client-4-api:4.5.10-2.0 -artifactory:3.6.2 -authentication-tokens:1.3 -aws-credentials:1.28 -aws-java-sdk:1.11.723 -blueocean-autofavorite:1.2.4 -blueocean-bitbucket-pipeline:1.23.2 -blueocean-commons:1.23.2 -blueocean-config:1.23.2 -blueocean-core-js:1.23.2 -blueocean-dashboard:1.23.2 -blueocean-display-url:2.3.1 -blueocean-events:1.23.2 -blueocean-git-pipeline:1.23.2 -blueocean-github-pipeline:1.23.2 -blueocean-i18n:1.23.2 -blueocean-jira:1.23.2 -blueocean-jwt:1.23.2 -blueocean-personalization:1.23.2 -blueocean-pipeline-api-impl:1.23.2 -blueocean-pipeline-editor:1.23.2 -blueocean-pipeline-scm-api:1.23.2 -blueocean-rest-impl:1.23.2 -blueocean-rest:1.23.2 -blueocean-web:1.23.2 -blueocean:1.23.2 -bouncycastle-api:2.18 -branch-api:2.5.6 -build-name-setter:2.1.0 -build-timestamp:1.0.3 -cloudbees-bitbucket-branch-source:2.8.0 -cloudbees-folder:6.12 -command-launcher:1.2 -config-file-provider:3.6.3 -convert-to-pipeline:1.0 -credentials-binding:1.23 -credentials:2.3.7 -custom-build-properties:1.9.1 -declarative-pipeline-migration-assistant-api:1.04 -declarative-pipeline-migration-assistant:1.04 -display-url-api:2.3.2 -docker-commons:1.16 -docker-java-api:3.1.5.2 -docker-plugin:1.2.0 -docker-workflow:1.23 -durable-task:1.34 -external-monitor-job:1.7 -favorite:2.3.2 -git-client:3.2.1 -git-server:1.9 -git:4.2.2 -github-api:1.111 -github-branch-source:2.7.1 -github:1.30.0 -gitlab-plugin:1.5.13 -gradle:1.36 -handlebars:1.1.1 -handy-uri-templates-2-api:2.1.8-1.0 -htmlpublisher:1.23 -ivy:2.1 -jackson2-api:2.11.0 -javadoc:1.5 -jdk-tool:1.4 -jenkins-design-language:1.23.2 -jira-steps:1.6.0 -jira:3.0.15 -jquery-detached:1.2.1 -jsch:0.1.55.2 -junit:1.29 -kubernetes-pipeline-devops-steps:1.6 -ldap:1.24 -lockable-resources:2.8 -mailer:1.32 -mapdb-api:1.0.9.0 -matrix-auth:2.6.1 -matrix-project:1.14 -maven-plugin:3.6 -mercurial:2.10 -metrics:4.0.2.6 -momentjs:1.1.1 -multibranch-action-triggers:1.8.5 -multibranch-build-strategy-extension:1.0.10 -pam-auth:1.6 -pipeline-as-yaml:0.3.0-rc -pipeline-aws:1.41 -pipeline-build-step:2.12 -pipeline-cps-http:0.2.1 -pipeline-dependency-walker:1.0.0 -pipeline-github-lib:1.0 -pipeline-githubnotify-step:1.0.5 -pipeline-graph-analysis:1.10 -pipeline-input-step:2.11 -pipeline-milestone-step:1.3.1 -pipeline-model-api:1.6.0 -pipeline-model-declarative-agent:1.1.1 -pipeline-model-definition:1.6.0 -pipeline-model-extensions:1.6.0 -pipeline-npm:0.9.2 -pipeline-rest-api:2.13 -pipeline-restful-api:0.6 -pipeline-stage-step:2.3 -pipeline-stage-tags-metadata:1.6.0 -pipeline-stage-view:2.13 -pipeline-timeline:1.0.3 -pipeline-utility-steps:2.5.0 -plain-credentials:1.7 -pubsub-light:1.13 -run-condition:1.3 -scm-api:2.6.3 -script-security:1.72 -simple-build-for-pipeline:0.2 -snakeyaml-api:1.26.4 -sse-gateway:1.23 -ssh-credentials:1.18.1 -ssh-slaves:1.31.2 -ssh-steps:2.0.0 -structs:1.20 -subversion:2.13.1 -timestamper:1.11.3 -token-macro:2.12 -translation:1.16 -trilead-api:1.0.6 -variant:1.3 -windows-slaves:1.6 -workflow-aggregator:2.6 -workflow-api:2.40 -workflow-basic-steps:2.20 -workflow-cps-global-lib-http:1.5.0 -workflow-cps-global-lib:2.16 -workflow-cps:2.80 -workflow-durable-task-step:2.35 -workflow-job:2.39 -workflow-multibranch:2.21 -workflow-scm-step:2.11 -workflow-step-api:2.22 -workflow-support:3.4 -job-dsl-plugin:1.77 -xml-job-to-job-dsl:0.1.13 diff --git a/jenpy/Taskfile.yml b/jenpy/Taskfile.yml new file mode 100644 index 0000000..96a06da --- /dev/null +++ b/jenpy/Taskfile.yml @@ -0,0 +1,17 @@ +# https://taskfile.dev + +version: '3' + +includes: + pyinst: ../tools/pyinst.tasks.yml + +vars: + GREETING: Hello, World! + +tasks: + default: + cmds: + - echo "{{.GREETING}}" + - task -l + silent: true +# https://pypi.org/project/junit-xml// \ No newline at end of file diff --git a/jenpy/_version/paths_config.yml b/jenpy/_version/paths_config.yml new file mode 100644 index 0000000..6db87ac --- /dev/null +++ b/jenpy/_version/paths_config.yml @@ -0,0 +1,3 @@ +repo_root_dir: .. +proj_root_dir: jenpy +proj_deps_dir: jenpy/deps diff --git a/jenpy/_version/pyinstaller_config.json b/jenpy/_version/pyinstaller_config.json new file mode 100644 index 0000000..ce1e2ef --- /dev/null +++ b/jenpy/_version/pyinstaller_config.json @@ -0,0 +1,7 @@ +{ + "entry_point": "jenpy_cli.py", + "exe_name": "jenpy", + "hiddenimports": [ + "gitlab.v4" + ] + } diff --git a/jenpy/deps/conda.base.deps.list.txt b/jenpy/deps/conda.base.deps.list.txt new file mode 100644 index 0000000..2ed3d41 --- /dev/null +++ b/jenpy/deps/conda.base.deps.list.txt @@ -0,0 +1,2 @@ +python=3.7 +pip diff --git a/jenpy/deps/conda.run.deps.list.txt b/jenpy/deps/conda.run.deps.list.txt new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/jenpy/deps/conda.run.deps.list.txt @@ -0,0 +1 @@ +pip diff --git a/jenpy/deps/pip.deps.list.txt b/jenpy/deps/pip.deps.list.txt new file mode 100644 index 0000000..00f746c --- /dev/null +++ b/jenpy/deps/pip.deps.list.txt @@ -0,0 +1,10 @@ +fire +loguru +python-jenkins +python-gitlab +waiting +coverage +pytest +pytest-cov +pytest-xdist + diff --git a/jenpy/jenlab_cli.py b/jenpy/jenlab_cli.py new file mode 100644 index 0000000..bef29c6 --- /dev/null +++ b/jenpy/jenlab_cli.py @@ -0,0 +1,42 @@ +import gitlab + +# private token or personal token authentication +gl = gitlab.Gitlab('http://localhost:33080', +private_token='zunNra1UfNtxfqggc9mu') + +gl.projects.list() + +group = gl.groups.create({ + 'name': 'jenlib_sample', + 'path': 'jenlib_sample' +}) + +group.visibility='public' +group.save() + +project = gl.projects.create({ + 'name': 'jenlib_sample_name', + 'visibility': 'public', + 'namespace_id': group.id +}) + +gl.projects.list() + +fine_name = 'build.pipe.groovy' +file_content =""" +node { + echo 'hello' + sh 'echo world' + stage('fetch'){ + def scmvars = checkout scm + } +} +""" + # 'encoding': 'text', + +f = project.files.create({'file_path': fine_name, + 'branch': 'master', + 'content': file_content, + 'author_email': 'test@example.com', + 'author_name': 'yourname', + 'commit_message': 'Create testfile'}) \ No newline at end of file diff --git a/jenpy/jenpy_cli.py b/jenpy/jenpy_cli.py new file mode 100755 index 0000000..4063351 --- /dev/null +++ b/jenpy/jenpy_cli.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python + +import fire +import jenkins +import time +# https://myopswork.com/trigger-multiple-jenkins-jobs-from-json-api-with-python-script-244ea85557a9 +# https://wiki.jenkins.io/display/JENKINS/Remote+access+API/ + + +# class JenBuild: + + +class JenCli: + + def __init__(self): + self.server = jenkins.Jenkins('http://jencondadev.samexample.com:8080', username='tester', password='tester') + + def info_sample(self): + server = self.server + + user = server.get_whoami() + version = server.get_version() + print('Hello %s from Jenkins %s' % (user['fullName'], version)) + + def build_bootstrap(self): + server = self.server + job_name = '_new_ci' + build_number = server.build_job(job_name, parameters=dict( + seed_job_branch='nextrelease.v0.7.6', + seed_job_name='_auto_seed') + ) + + def build_job_block(self, j_name, parameters=None, show_log=False): + # https://stackoverflow.com/questions/24313471/python-jenkins-get-job-info-how-do-i-get-info-on-more-than-100-builds + server = self.server + ps = {} + if parameters: + ps['parameters'] = parameters + + next_build_number = server.get_job_info(j_name)['nextBuildNumber'] + gbuild = server.build_job(j_name, **ps) + while True: + print(f'Waiting....{gbuild} as {next_build_number} for {j_name}') + try: + if server.get_job_info(j_name)['lastCompletedBuild']['number'] == server.get_job_info(j_name)['lastBuild']['number']: + print ("Last ID %s, Current ID %s" % (server.get_job_info(j_name)['lastCompletedBuild']['number'], server.get_job_info(j_name)['lastBuild']['number'])) + break + except: + pass + time.sleep(2) + print('Stop....') + last_number = server.get_job_info(j_name)['lastBuild']['number'] + jobinfo = server.get_build_info(j_name, last_number) + _ret = { + 'jobinfo': jobinfo, + 'gbuild': gbuild, + 'jobname': j_name, + 'last_number': last_number, + + 'console_cmd': f'server get_build_console_output {j_name} {last_number}' + } + if show_log: + pet ={} + pet['console_log'] = server.get_build_console_output(j_name, last_number) + print(pet['console_log']) + # print(_ret) + return _ret + + def build_sample(self, show_log=False): + return self.build_job_block('samples/pipe_501_cmds_explicit.groovy', show_log=show_log) + + def test_bootstrap(self,show_log=1): + print("asdf") + server = self.server + import subprocess + dt = subprocess.check_output(["date","+%Y%m%d.%H%M%S"]).decode().strip() + + + import waiting + waiting.wait( + lambda: server.get_job_info('_new_ci'), + timeout_seconds=15, + expected_exceptions=(jenkins.JenkinsException,) + ) + seed_job = f'_test_seed_{dt}' + _ret = self.build_job_block( + '_new_ci', + parameters=dict( + seed_job_branch='nextrelease.v0.7.6', + seed_job_name=seed_job), + show_log=show_log, + ) + print(_ret) + # wait until job is created + print("@@act=wait stage=start topic='job created'") + waiting.wait( + lambda: server.get_job_info(seed_job), + timeout_seconds=10, + expected_exceptions=(jenkins.JenkinsException,) + ) + print("@@act=wait stage=over topic='job created'") + # time.sleep(15) + _ret = self.build_job_block(seed_job, show_log=show_log) + print(_ret) + + +if __name__ == '__main__': + # jc = JenCli() + # jc.build_sample() + fire.Fire(JenCli) diff --git a/jenpy/pyinstaller.spec b/jenpy/pyinstaller.spec new file mode 100644 index 0000000..da3bd13 --- /dev/null +++ b/jenpy/pyinstaller.spec @@ -0,0 +1,53 @@ +# -*- mode: python ; coding: utf-8 -*- + +# from.url=https://geekswithlatitude.readme.io/docs/create-exe-with-data-files + +from pathlib import Path +import json + +block_cipher = None + +my_dir = Path('.') +config_path = my_dir / '_version' / 'pyinstaller_config.json' +# doc = yaml.load(config_path.read_text().strip(), Loader=yaml.FullLoader) +doc = json.loads(config_path.read_text().strip()) + +entry_point_path = doc['entry_point'] +exe_name = doc['exe_name'] +hiddenimports = doc.get('hiddenimports', []) + +a = Analysis([entry_point_path], + pathex=['.'], + binaries=[], + datas=[], + hiddenimports=hiddenimports, + # hiddenimports=['pyspark'], + hookspath=[], + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False) +pyz = PYZ(a.pure, a.zipped_data, + cipher=block_cipher) + +# a.datas += [ +# ('version/version_prefix.txt', 'version/version_prefix.txt', 'DATA'), +# ('version/project_prefix.txt', 'version/project_prefix.txt', 'DATA'), +# ] + +exe = EXE(pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + [], + name=exe_name, + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir=None, + console=True ) diff --git a/jenpy/rep_junit.py b/jenpy/rep_junit.py new file mode 100644 index 0000000..e69de29 diff --git a/jenpy/sample_junit_out.xml b/jenpy/sample_junit_out.xml new file mode 100644 index 0000000..094354e --- /dev/null +++ b/jenpy/sample_junit_out.xml @@ -0,0 +1,21 @@ + + + + + + I am stdout! + + + I am stderr! + + + + + I am stdout! + + + I am stderr! + + + + diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..328a1a9 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,7 @@ +site_name: My Docs +theme: material +dev_addr: 0.0.0.0:8690 +site_dir: __localbuild__/site +nav: + - Home: index.md + - Release Notes: release_notes.md diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..6167ce9 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,50 @@ +[tool:pytest] +norecursedirs = + docs/* + .history + .git + samples + __localbuild__ + __build_info + _infra + infra + data4test + __mxp_version.py + +; python_files = *__test.py +; python_functions = test__* +; log_cli=true +log_cli = true +log_cli_level = INFO +; log_cli_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s) +; log_cli_date_format=%Y-%m-%d %H:%M:%S +; capture = tee-sys + +[coverage:run] +## parallel coverage +# @@from.url=https://stackoverflow.com/questions/61759065/for-pytest-with-pytest-cov-how-to-specify-parallel-true-for-coverage-version +omit = + # omit anything in a .local directory anywhere + conftest.py + */__init__.py + tests/* + */site-packages/* + */__mxp_version.py + # omit everything in /usr + #/usr/* + # omit this single file + #utils/tirefire.py + +[yapf] +; [style] +# based_on_style = pep8 +based_on_style = google +# PEP8 Line Width is 79; set bigger for data science +column_limit = 100 +split_arguments_when_comma_terminated = true +split_before_logical_operator = true +# split_all_comma_separated_values = true +dedent_closing_brackets = true +# force_multiline_dict = true +# allow_split_before_dict_value = false +# split_before_bitwise_operator = false diff --git a/src/Taskfile.yml b/src/Taskfile.yml new file mode 100644 index 0000000..4c1dafc --- /dev/null +++ b/src/Taskfile.yml @@ -0,0 +1,35 @@ +# https://taskfile.dev + +version: '3' + +# includes: +# basic: tasks/basic.tests.tasks.yml +vars: + GREETING: Hello, World! + +tasks: + default: + cmds: + - echo "{{.GREETING}}" + - task -l + silent: true + + test: + desc: run all tests + cmds: + - groovy run_tests.groovy tests=all + + test-one: + desc: run all tests + args: + test: str. name of test to run + vars: + test_val: '{{ default "test_stages_from_task" .test}}' + cmds: + - groovy run_tests.groovy tests={{.test_val}} + + + list-tests: + desc: run all tests + cmds: + - groovy run_tests.groovy list-tests \ No newline at end of file diff --git a/src/lib/Lib.groovy b/src/lib/Lib.groovy new file mode 100644 index 0000000..b149055 --- /dev/null +++ b/src/lib/Lib.groovy @@ -0,0 +1,5 @@ + package lib + class Lib { + static saySomething() { println 'something' } + def sum(a,b) { a+b } + } diff --git a/src/lib/old_tapp.groovy b/src/lib/old_tapp.groovy new file mode 100644 index 0000000..70e1b4a --- /dev/null +++ b/src/lib/old_tapp.groovy @@ -0,0 +1,69 @@ + +@Grab('org.yaml:snakeyaml:1.17') +import org.yaml.snakeyaml.Yaml + +import lib.Lib +import org.jenslib.JenApi +import org.jenslib.Jenapi +import tests.unit.TestsBasic + +def apis = [ + lib: new Lib(), + jenapi: new JenApi(), + jenapi_alt: new Jenapi() +] + +def test_sample(def apis){ + Lib.saySomething(); + println new Lib().sum(37,5) + println new JenApi().reverse_string("it") + def jap = new Jenapi() + println jap.reverse_string("on") + def tb = new TestsBasic() + + tb.test_read_task() + def methods = TestsBasic.declaredMethods.findAll { !it.synthetic && it.name.startsWith("test")}.name + println methods +} + + +test_sample(apis) + + + +// def runner(def kwargs){ +// if (kwargs.length < 2){ +// println "Usage: groovy-run [method]:req METHOD_NAME" +// return +// } +// File sourceFile = new File("/homes/yairda/_wd/mx-infra/jenlib/src/org/jenslib/jentools.groovy"); +// Class groovyClass = new GroovyClassLoader(getClass().getClassLoader()).parseClass(sourceFile); +// GroovyObject myObject = (GroovyObject) groovyClass.newInstance(); +// // StaticApi +// // https://stackoverflow.com/questions/9136328/including-a-groovy-script-in-another-groovy +// // evaluate implicitly creates a class based on the filename specified + +// // evaluate(new File("./Testutils.groovy")) +// // // Safer to use 'def' here as Groovy seems fussy about whether the filename (and therefore implicit class name) has a capital first letter +// // def tu = new Testutils() +// // tu.myUtilityMethod("hello world") + +// // if kwargs[0] == 'method' +// // from=http://www.groovy-lang.org/metaprogramming.html#_dynamic_method_names +// // def api = myObject //new StaticApi() +// def api = myObject // new .StaticApi() //new StaticApi() +// def method = kwargs[1] +// def argstr = kwargs[2] +// def now_ts= new Date().format( 'yyyyMMdd-hhmm-ss00' ) +// def uid = UUID.randomUUID().toString() +// def suid = uid[19..-1] +// def runid = "${now_ts}-${suid}" +// println "${method} ${argstr}" + +// println "@@act=prep target=test runid=${runid} name=${method} argstr=${argstr} agent=root.groovy.test.runner" +// def result = api."${method}"(argstr) +// println "@@act=post target=test runid=${runid} name=${method} result=${result} agent=root.groovy.test.runner" +// } + +// // args.each{println it} +// runner(args) diff --git a/src/org/jenslib/JenApi.groovy b/src/org/jenslib/JenApi.groovy new file mode 100644 index 0000000..42876d6 --- /dev/null +++ b/src/org/jenslib/JenApi.groovy @@ -0,0 +1,46 @@ +package org.jenslib + +@Grab('org.yaml:snakeyaml:1.17') +import org.yaml.snakeyaml.Yaml + +def reverse_string(String s){ + return s.reverse() +} + +def checkOutFrom(repo) { + sh(" echo git@github.com:jenkinsci/${repo}") +} + +def text_yaml_to_map(text){ + def yaml_parser = new Yaml() + def parsed = yaml_parser.load(text) + return parsed +} + +def text_yaml_from_map(the_map){ + def yaml_parser = new Yaml() + def text = yaml_parser.dump(the_map) + return text +} + +def stages_from_task(task_map){ + for (cmd in task_map.cmds){ + stage(cmd){ + sh(cmd) + } + } +} + +def stage_with_steps_from_task(Map cfg =[:]){ + def taskmap = cfg.taskmap; assert taskmap + def taskname = cfg.taskname; assert taskname + // end of header schema + + stage(taskname){ + for (cmd in taskmap.cmds){ + sh(cmd) + } + } +} + +return this diff --git a/src/org/jenslib/Jenapi.groovy b/src/org/jenslib/Jenapi.groovy new file mode 100644 index 0000000..a37067a --- /dev/null +++ b/src/org/jenslib/Jenapi.groovy @@ -0,0 +1,11 @@ +package org.jenslib + +class Jenapi{ + public static String reverse_string(String s){ + return s.reverse() + } + + def process_taskfile(text){ + + } +} diff --git a/src/org/jenslib/Utilities.groovy b/src/org/jenslib/Utilities.groovy new file mode 100644 index 0000000..ba407d6 --- /dev/null +++ b/src/org/jenslib/Utilities.groovy @@ -0,0 +1,16 @@ +package org.jenslib + +class Utilities implements Serializable { + def steps + Utilities(steps) {this.steps = steps} + def mvn(args) { + steps.sh "${steps.tool 'Maven'}/bin/mvn -o ${args}" + } +} + +// @Library('utils') import org.foo.Utilities +// def utils = new Utilities(this) +// node { +// utils.mvn 'clean package' +// } +// return this diff --git a/src/org/jenslib/dotenv.groovy b/src/org/jenslib/dotenv.groovy new file mode 100644 index 0000000..330c7b1 --- /dev/null +++ b/src/org/jenslib/dotenv.groovy @@ -0,0 +1,81 @@ +package org.jenslib + +// --- docs --- + +// https://itnext.io/jenkins-shared-libraries-part-1-5ba3d072536a +// https://tomd.xyz/jenkins-shared-library/ +// https://www.jenkins.io/doc/book/pipeline/shared-libraries/ +// https://groovy-playground.appspot.com/ +// https://groovy-lang.gitlab.io/101-scripts/docker/basico-en.html +// https://github.com/groovy/docker-groovy + + +// --- yaml sample --- +// @Grab('org.yaml:snakeyaml:1.17') +// import org.yaml.snakeyaml.Yaml + +// --- private --- + +def remove_quotes(txt){ + if (txt[0] == txt[-1]){ + // println "same" + if (txt[0] == "'" || txt[0] == '"') { + // println txt.substring(1,txt.length()-1) + return txt.substring(1,txt.length()-1) + } + } + return txt +} + +def parse_groups(env_file_conf, line){ + def full_var_name = line.takeWhile { it != '=' } + def var_content_raw = line.substring(full_var_name.length()+1) + def var_content_clean = remove_quotes(var_content_raw) + + def group_name = full_var_name.takeWhile { it != '_' } + def pack_name_long = full_var_name.substring(group_name.length()+1) + println ( [group_name, pack_name_long] ) + + def pack_name = pack_name_long.split('__')[0] + def var_name_clean = pack_name_long.split('__')[1] + + println ( [pack_name, var_name_clean] ) + + env_file_conf.full << ["${full_var_name}": "${var_content_clean}"] + env_file_conf.vars << ["${var_name_clean}": "${var_content_clean}"] + if(! env_file_conf.packs[pack_name]){ + env_file_conf.packs[pack_name] = [:] + } + env_file_conf.packs[pack_name] << ["${var_name_clean}": "${var_content_clean}"] + if(! env_file_conf.groups){ + env_file_conf.groups[group_name] = [:] + } + env_file_conf.groups[group_name] << ["${pack_name_long}": "${var_content_clean}"] +} + +def parse_env_file(multiline_text){ + def lines = multiline_text.readLines() + def mxpack_lines = lines.collectMany { + (it.trim() =~ /^MXPACK_.*/) + ? [it.trim()] + : [] + } + def env_file_conf = [ + full: [:], + groups: [ + MXPACK: [:] + ], + vars: [:], + packs: [:] + ] + mxpack_lines.each { it -> parse_groups(env_file_conf, it)} + return env_file_conf +} + +// --- public --- + +class jenlib { + static def parse_env_file_text(env_file_text) { + return parse_env_file(env_file_text) + } +} diff --git a/src/org/jenslib/jentools.groovy b/src/org/jenslib/jentools.groovy new file mode 100644 index 0000000..1e3b2c8 --- /dev/null +++ b/src/org/jenslib/jentools.groovy @@ -0,0 +1,30 @@ +package org.jenslib.jentools + +class StaticApi{ + public static String reverse_string(String s){ + return s.reverse() + } +} + +def reverse_string(String s){ + return s.reverse() +} + +def checkOutFrom(repo) { + sh(" echo git@github.com:jenkinsci/${repo}") +} + +class Utilities implements Serializable { + def steps + Utilities(steps) {this.steps = steps} + def mvn(args) { + steps.sh "${steps.tool 'Maven'}/bin/mvn -o ${args}" + } +} + +// @Library('utils') import org.foo.Utilities +// def utils = new Utilities(this) +// node { +// utils.mvn 'clean package' +// } +return this diff --git a/src/org/jenslib/z_old.mxpack_lib.groovy b/src/org/jenslib/z_old.mxpack_lib.groovy new file mode 100644 index 0000000..736a2c9 --- /dev/null +++ b/src/org/jenslib/z_old.mxpack_lib.groovy @@ -0,0 +1,106 @@ +text=""" +# This is .env file with all settings for all packages +# notation: +# MXPACK_{PACKGROUP}__{VARIABLE} + +# --- version --- +MXPACK_MXPACK__VERSION_SCHEMA="0.14.1331" + +# --- mxpack global config --- +MXPACK_MXPACK__LOCAL_BUILD_DIR=__localbuild__ +MXPACK_MXPACK__LOCAL_BUILD_INFO=__build_info + +# --- mxpack project pack config --- +MXPACK_PROJECT__PROJECT_NAME=objx-homepage + +# --- mxpack gitdata pack config --- +MXPACK_GITDATA__GIT_TAG_PREFIX=objx-homepage- + +# --- mxpack mkdocs pack config --- +MXPACK_MKDOCS__VENV_NAME="objx-homepage-venv-py37" +MXPACK_MKDOCS__PY_VERSION="3.7" + +# --- mxpack jenlib pack config --- +MXPACK_MKDOCS__JENLIB_SETTINGS_YML=version/recipes/jenlib.pack.settings.yml + +""" + +// println text +println text[0] + +def remove_quotes(txt){ + if (txt[0] == txt[-1]){ + // println "same" + if (txt[0] == "'" || txt[0] == '"') { + // println txt.substring(1,txt.length()-1) + return txt.substring(1,txt.length()-1) + } + } + return txt +} +println( remove_quotes('"noqouetes"') ) + +def parse_flat(line){ + def var_name = line.takeWhile { it != '=' } + def var_content_raw = line.substring(var_name.length()+1) + def var_content_clean = remove_quotes(var_content_raw) + return +} +println parse_flat("""MXPACK_MXPACK__VERSION_SCHEMA='0.14.1331'""") +println parse_flat("""MXPACK_MXPACK__LOCAL_BUILD_DIR=__localbuild__""") + + +def parse_groups(kw, line){ + def full_var_name = line.takeWhile { it != '=' } + def var_content_raw = line.substring(full_var_name.length()+1) + def var_content_clean = remove_quotes(var_content_raw) + + def group_name = full_var_name.takeWhile { it != '_' } + def pack_name_long = full_var_name.substring(group_name.length()+1) + println ( [group_name, pack_name_long] ) + + def pack_name = pack_name_long.split('__')[0] + def var_name_clean = pack_name_long.split('__')[1] + + println ( [pack_name, var_name_clean] ) + + kw.full << ["${full_var_name}": "${var_content_clean}"] + kw.vars << ["${var_name_clean}": "${var_content_clean}"] + if(! kw.packs[pack_name]){ + kw.packs[pack_name] = [:] + } + kw.packs[pack_name] << ["${var_name_clean}": "${var_content_clean}"] + if(! kw.groups){ + kw.groups[group_name] = [:] + } + kw.groups[group_name] << ["${pack_name_long}": "${var_content_clean}"] + + +} + +def parse_env_file(multiline_text){ + def lines = multiline_text.readLines() + def mxpack_lines = lines.collectMany { + (it.trim() =~ /^MXPACK_.*/) + ? [it.trim()] + : [] + } + def kw = [ + full: [:], + groups: [ + MXPACK: [:] + ], + vars: [:], + packs: [:] + ] + mxpack_lines.each { it -> parse_groups(kw, it)} + + return kw + +} +import static groovy.json.JsonOutput.* + +def config = parse_env_file(text) + +println prettyPrint(toJson(config)) + diff --git a/src/run_tests.groovy b/src/run_tests.groovy new file mode 100644 index 0000000..c447107 --- /dev/null +++ b/src/run_tests.groovy @@ -0,0 +1,197 @@ + +@Grab('org.yaml:snakeyaml:1.17') +import org.yaml.snakeyaml.Yaml + +import lib.Lib +import org.jenslib.JenApi +import org.jenslib.Jenapi +import tests.unit.TestsBasic + + +def get_jenapi(){ + // TODO: move to file + def org_ast_holder= [root:[:], path: '.', cur: null] + JenApi.metaClass.ast_holder = org_ast_holder + + JenApi.metaClass.reset_ast = { + println "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + ast_holder.root = [ + stages: [] + ] + ast_holder.cur = ast_holder.root.stages + } + + JenApi.metaClass.sh ={ + name -> println "JEN_SH ${name}" + def step = [ + step: "sh", + mparams:[ + script: name + ] + ] + ast_holder.cur.add(step) + } + + JenApi.metaClass.stage ={ + name, body -> println "JEN_STAGE ${name}" + def stage = [ + stage: name, + steps:[] + ] + def pcur = ast_holder.cur + ast_holder.cur.add(stage) + ast_holder.cur = stage.steps + body() + ast_holder.cur = pcur + } + + + def jenapi_obj = new JenApi() + return jenapi_obj + +} + +def get_data4test_dir(){ + def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).parent + def repo_root = new File(scriptDir).parent + def data4test_dir = "${repo_root}/data4test/jenlib_pure" + return data4test_dir + + // //One more solution. It works perfect even you run the script using GrovyConsole + + // File getScriptFile(){ + // new File(this.class.classLoader.getResourceLoader().loadGroovySource(this.class.name).toURI()) + // } + + // println getScriptFile() + +} + +def tenv = [ + lib: new Lib(), + jenapi: get_jenapi(), + jenapi_alt: new Jenapi(), + test_suits: [ + basic: new TestsBasic() + ], + data4test_dir: get_data4test_dir() +] + +def test_sample(def tenv){ + tenv.lib.saySomething(); + println tenv.lib.sum(37,5) + println tenv.jenapi.reverse_string("it") + println tenv.jenapi_alt.reverse_string("on") + + println get_data4test_dir() +} + +def _print_usage(String err=""){ + println "Usage: groovy run_tests.groovy tests=all" + println "Usage: groovy run_tests.groovy tests=test1,test2" + if (err){ + println err + } + return +} + +def discover_tests(kwargs, tenv){ + def holder = tenv.test_suits.basic + def api = tenv.jenapi + + if (kwargs.length == 0){ + return {_print_usage()} + } + if (kwargs.length == 1) { + def first_arg = kwargs[0] + + def methods = holder.getClass().declaredMethods.findAll { + !it.synthetic && it.name.startsWith("test") + }.name + + methods = methods.unique() + def mc = methods.getClass() + + if (first_arg=='list-tests'){ + println "Abailable Tests" + return { + for( m in methods ){ + println m + } + } + } + if (first_arg=='test-sample'){ + println "Running Smaple Test" + return { + test_sample(tenv) + } + } + + def pair_list = first_arg.split('=') as List + if (pair_list.size() != 2){ + def err = "Args should be in X=Y pairs, but got: ${first_arg}" + return {_print_usage(err)} + } + + assert pair_list[0] == 'tests' + + def tests_to_run = pair_list[1].split(',') + if (tests_to_run.length == 0){ + def err = """ + Args should be in tests=t1,t2 format, + t1,t2 is list of test and = it should be coma separated + got ${first_arg} + """ + return {_print_usage(err)} + } else { + if (tests_to_run[0] == 'all'){ + + tests_to_run = methods + } + + return {run_many_tests(api, tenv,holder, tests_to_run)} + } + // api: tenv.jenapi + } +} + +def run_many_tests(api, tenv, holder, methods){ + println "api=${api}. methods=${methods} " + def mc = methods.getClass() + + println("methods: ${mc}") + + for (var i = 0; i < methods.size(); i++){ + def the_method = methods[i] + println "SDFSD=${the_method}" + // throw new Exception("sadf") + run_one_test(api, tenv, holder, the_method) + } +} + +def run_one_test(api, tenv, holder, method, argstr=''){ + // argstr - not in use + api.reset_ast() + + def now_ts= new Date().format( 'yyyyMMdd-hhmm-ss00' ) + def uid = UUID.randomUUID().toString() + def suid = uid[19..-1] + def runid = "${now_ts}-${suid}" + println "${method} ${argstr}" + + println "@@act=prep target=test runid=${runid} name=${method} argstr=${argstr} agent=root.groovy.test.runner" + def result = holder."${method}"( + api: api, + tenv: tenv + ) + println "@@act=post target=test runid=${runid} name=${method} result=${result} agent=root.groovy.test.runner" +} + +def runner(def kwargs, def tenv){ + println "conf: ${kwargs}" + def func = discover_tests(kwargs, tenv) + return func() +} + +// args.each{println it} +runner(args, tenv) diff --git a/src/tasks/Taskfile.yml b/src/tasks/Taskfile.yml new file mode 100644 index 0000000..7a86953 --- /dev/null +++ b/src/tasks/Taskfile.yml @@ -0,0 +1,14 @@ +# https://taskfile.dev + +version: '3' +includes: + tbasic: test-basic.tasks.yml +vars: + GREETING: Hello, World! + +tasks: + default: + cmds: + - echo "{{.GREETING}}" + - task -l + silent: true diff --git a/src/tasks/basic.tests.tasks.yml b/src/tasks/basic.tests.tasks.yml new file mode 100644 index 0000000..3fed2bc --- /dev/null +++ b/src/tasks/basic.tests.tasks.yml @@ -0,0 +1,52 @@ +# https://taskfile.dev + +version: '3' + +vars: + GREETING: Hello, World! + _out_dir: _out/grv_tests + _read_cfg: yq r basic.tests.tasks.yml ${YQPATH} | sed 's/:[ ]/=/' + +confs: + check-output: + reverse-str: + method: reverse_string + argstr: "awesome" + expected: "emosewa" + +tasks: + default: + cmds: + - echo "{{.GREETING}}" + silent: true + + test-groovy-output: &ref_test_groovy_output + desc: _ + connector: name=groovy + args: + method: required=true desc=method to run + argstr: required=false desc='argument string' + vars: + run_id: "$(date +'%Y%m%d%H-%M%S-0000-0000')" + run_log: "_out/grv_tests/{{.run_id}}-{{.TASK}}.rid.txt" + cmds: + - echo {{.run_id}} | rclone rcat {{.run_log}} + - | + groovy ../run_tests.groovy \ + unit/test_basic.groovy method \ + {{.method}} {{.argstr}} \ + | tee {{.run_log}} + - cat {{.run_log}} | grep {{.expected}} + + + test-reverse-string: + <<: *ref_test_groovy_output + vars: + method: reverse_string + argstr: "awesome" + expected: "emosewa" + + run-all: + desc: _ + cmds: + - task: test-reverse-string diff --git a/src/tasks/cli_task.tests.tasks.yml b/src/tasks/cli_task.tests.tasks.yml new file mode 100644 index 0000000..3fed2bc --- /dev/null +++ b/src/tasks/cli_task.tests.tasks.yml @@ -0,0 +1,52 @@ +# https://taskfile.dev + +version: '3' + +vars: + GREETING: Hello, World! + _out_dir: _out/grv_tests + _read_cfg: yq r basic.tests.tasks.yml ${YQPATH} | sed 's/:[ ]/=/' + +confs: + check-output: + reverse-str: + method: reverse_string + argstr: "awesome" + expected: "emosewa" + +tasks: + default: + cmds: + - echo "{{.GREETING}}" + silent: true + + test-groovy-output: &ref_test_groovy_output + desc: _ + connector: name=groovy + args: + method: required=true desc=method to run + argstr: required=false desc='argument string' + vars: + run_id: "$(date +'%Y%m%d%H-%M%S-0000-0000')" + run_log: "_out/grv_tests/{{.run_id}}-{{.TASK}}.rid.txt" + cmds: + - echo {{.run_id}} | rclone rcat {{.run_log}} + - | + groovy ../run_tests.groovy \ + unit/test_basic.groovy method \ + {{.method}} {{.argstr}} \ + | tee {{.run_log}} + - cat {{.run_log}} | grep {{.expected}} + + + test-reverse-string: + <<: *ref_test_groovy_output + vars: + method: reverse_string + argstr: "awesome" + expected: "emosewa" + + run-all: + desc: _ + cmds: + - task: test-reverse-string diff --git a/src/tasks/tasks.inline-exec.tasks.yml b/src/tasks/tasks.inline-exec.tasks.yml new file mode 100644 index 0000000..3ed7eb5 --- /dev/null +++ b/src/tasks/tasks.inline-exec.tasks.yml @@ -0,0 +1,55 @@ +# https://taskfile.dev + +version: '3' + +includes: + tbasic: test-basic.tasks.yml +vars: + GREETING: Hello, World! + +tasks: + default: + cmds: + - echo "{{.GREETING}}" + silent: true + + try-groovy: + desc: _ + vars: + var: A.B.C + cmds: + - | + groovy -e ' + println "{{.var}}".split("\\.")[1..-1] + ' + + + try-py: + desc: _ + vars: + var: A.B.C + cmds: + - | + python -c ' + print("{{.var}}".split(".")[1:]) + ' + + + try-all: + desc: _ + vars: + env: + var: A.B.C + cmds: + - task: bash + vars: + cmd: ls + + - task: python + vars: + cmd: print("{{.var}}".split(".")[1:]) + + - task: groovy + vars: + cmd: print("{{.var}}".split(".")[1:]) + diff --git a/src/tasks/test-basic.tasks.yml b/src/tasks/test-basic.tasks.yml new file mode 100644 index 0000000..7a2009b --- /dev/null +++ b/src/tasks/test-basic.tasks.yml @@ -0,0 +1,216 @@ +# https://taskfile.dev + +version: '3' + +# includes: +# basic: basic.tests.tasks.yml +vars: + GREETING: Hello, World! + tasknamepref: tbasic + +tasks: + default: + cmds: + - echo "{{.GREETING}}" + silent: true + + # --- Jenkins Top Level Job Descriptors Tasks --- + + ci:ci_full: + desc: imaginary level mapped manually + cmds: + - task: ci:ci-begin + - task: ci:ci-flow + - task: ci:ci-over + + ci:ci-begin: + desc: preparing tasks + jenlib: {cmds_expand: true} + cmds: + - task: resolve-deps + + ci:ci-over: + desc: reporting publishing tasks + jenlib: {cmds_expand: true} + cmds: + - task: publish-arts + + ci:ci-flow: + desc: _ + vars: + _jenlib_: expand=all + cmds: + - task: compile-confs + - task: compile-confs2 + + resolve-deps: + desc: _ + cmds: + - echo {{.TASK}} + + # --- simple commands ---- + fmt: echo "it={{.item}}" + + compile-confs: + desc: _ + cmds: + - task: with-items + vars: {taskname: fmt, items: one two three} + + + compile-confs2: + desc: _ + cmds: + - task: with-items + vars: + taskname: fmt2 + task_opt: '-o prefixed' + items: "$(echo $(seq 7))" + jen_multi: true + + compile-confs3: + desc: _ + cmds: + - task: multi-bash + vars: + with_items: "$(echo $(seq 7))" + ITEM: '%' + cmd: "task {{ .tasknamepref}}:fmt item='%' -o prefixed" + + publish-arts: + desc: _ + cmds: + - echo {{.TASK}} + - echo asf + - echo {{.TASK}} + + fmt2: + cmds: + - seq {{.item}} | xargs -I% bash -c 'sleep 1 && echo {{.item}} %' + - |- + for i in `seq {{.item}}`; do + sleep 1 + echo [{{.item}}:$i] + done + # - task: sh + # vars: {cmd: ls, _label: open} + + - task resolve-deps + + prefix: "{{.TASK}}:{{.item}}" + + + + ci:ci-flow2: + desc: _ + vars: + _jenlib_: expand=all + cmds: + # - task: with-items + - task: resolve-stream + vars: + task: fmt + item_from: echo 'one two three' + item_type: tokens + + - task: resolve-stream + vars: + task: fmt + item_from: yq read test-basic.tasks.yml tasks + item_type: list-entry + item_convert: task-vars + - task with-items task=fmt items="echo 'one two three'" + - | + task with-items task=fmt items=echo ' + "one" + "two" + "three" + ' + - task: with-items + vars: + task: fmt + item_type: line + with_items_lines: | + one + two + three + jenlib_expand: expand=full + - echo [task={{.TASK}}] [state=DONE] + + # --- Jenkins Top Level Job Descriptors Tasks --- + + prepare: echo ls + + resolve-one: | + seq 5 | xargs -I% bash -c 'sleep 1 && echo {{.arg}} %' + + resolve-steps: + - echo [1] && sleep 1 + - echo [2] && sleep 1 + - echo [3] && sleep 1 + + multi: + desc: _ + deps: + - task: resolve-one + vars: {arg: one} + - task: resolve-steps + vars: {arg: two} + - task: resolve-one + vars: {arg: three} + cmds: + - echo [task={{.TASK}}] [state=DONE] + + multi2: + desc: _ + cmds: + + + + report: + desc: _ + cmds: + - echo report + + with-items: + # jenlib: {multitask: true, items: with_items, parallel: unfold, subseq: unfold} + args: + taskname: TASK_NAME + items: asdf + vars: + _ITEM: "{}" + cmds: + - task: multi-bash + vars: + with_items: "{{.items}}" + ITEM: '{{._ITEM}}' + cmd: "task {{.task_opt}} {{.taskpath_arg}} {{.tasknamepref}}:{{.taskname}} item='{{._ITEM}}'" + # jen_impl: | + # jinMakeParallel + + multi-bash: + # taskhub: + # url: http://taswkfile + # global_id: c129484-6e4749c + # version: 8 + + _desc: run multiple inputs in parallel + args: + cmd: command + with_items: '[cmd({}) for {} in with_items]' + XITEM: required=true suggested='{}' + vars: + jenlib: support=expand + count_intems: "$(echo {{.with_items}} | wc -w)" + cmds: + - echo {{.with_items} } + - echo "count_intems={{.count_intems}}" + - | + echo "{{.with_items}}" \ + | xargs -n 1 \ + | xargs \ + -I {{.ITEM}} \ + -P {{.count_intems}} \ + bash -c "{{.cmd}}" + prefix: "par:{{.count_intems}}" + # silent: true \ No newline at end of file diff --git a/src/tests/spock.groovy b/src/tests/spock.groovy new file mode 100644 index 0000000..e5e2d5d --- /dev/null +++ b/src/tests/spock.groovy @@ -0,0 +1,19 @@ +@Grab('org.spockframework:spock-core:1.2') + +class TestX extends spock.lang.Specification { + def 'test method'() { + } +} + +class HelloSpock extends spock.lang.Specification { + def "length of Spock's and his friends' names"() { + expect: + name.size() == length + + where: + name | length + "Spock" | 5 + "Kirk" | 4 + "Scotty" | 6 + } +} \ No newline at end of file diff --git a/src/tests/unit/TestsBasic.groovy b/src/tests/unit/TestsBasic.groovy new file mode 100644 index 0000000..3fce769 --- /dev/null +++ b/src/tests/unit/TestsBasic.groovy @@ -0,0 +1,163 @@ +package tests.unit + +@Grab('org.yaml:snakeyaml:1.17') +import org.yaml.snakeyaml.Yaml + + +def test_reverse_string(Map cfg=[:]){ + assert cfg.api + def api = cfg.api + + the_string = "someone" + expected = "enoemos" + actual = api.reverse_string(the_string) + assert actual == expected + return "state:success" +} + +def sample(){return """ +# https://taskfile.dev + +version: '3' + +GREETING: Hello, World! + +tasks: + default: + cmds: + - echo "{{.GREETING}}" + silent: true + + prepare: + cmds: + - pwd + + report: + cmds: + - ls -alh + + ci-flow: + cmds: + - echo ci-flow init + - echo ci-flow over + + top-level: + desc: _ + cmds: + - task prepare + - task ci-flow + - task report +""" +} + +def test_read_task(Map cfg=[:]){ + assert cfg.api + def api = cfg.api + + def text = sample() + def yaml_map = api.text_yaml_to_map(text) + assert yaml_map.version == '3' + println yaml_map +} + +def test_stages_from_task_inline(Map cfg=[:]){ + assert cfg.api + def api = cfg.api + def text = sample() + api.reset_ast() + def yaml_map = api.text_yaml_to_map(text) + def the_task = yaml_map.tasks['top-level'] + def stages = api.stages_from_task(the_task) + println stages + println api.text_yaml_from_map (api.ast_holder) + def actual = api.ast_holder.root.stages + def expected_txt = """ +- stage: task prepare + steps: + - step: sh + mparams: {script: task prepare} +- stage: task ci-flow + steps: + - step: sh + mparams: {script: task ci-flow} +- stage: task report + steps: + - step: sh + mparams: {script: task report}""" + + def expected = api.text_yaml_to_map(expected_txt) + assert actual == expected +} + + +def test_this_and_owner(Map cfg=[:]){ + def x = { def y = { println "this=";println this; println "owner="; println owner }; y() } + x() +} + + +def test_stages_from_task(Map cfg=[:]){ + assert cfg.api + def api = cfg.api + def tenv = cfg.tenv + + + def taskfile_path = "${tenv.data4test_dir}/test_basic_task/Taskfile.yml" + def jenflow_path ="${tenv.data4test_dir}/test_basic_task/Taskfile.jenflow.yml" + + def taskfile_text = new File(taskfile_path).text + def jenflow_text = new File(jenflow_path).text + + + def yaml_map = api.text_yaml_to_map(taskfile_text) + def the_task = yaml_map.tasks['top-level'] + def stages = api.stages_from_task(the_task) + + def actual = api.ast_holder.root.stages + + + def flow_full = api.text_yaml_to_map(jenflow_text) + def flow_map = flow_full.entries.stages_from_task + + def expected = flow_map.stages + + println api.text_yaml_from_map(actual) + + assert actual == expected +} + + +def stage_with_steps_from_task(Map cfg=[:]){ + assert cfg.api + def api = cfg.api + def tenv = cfg.tenv + + + def taskfile_path = "${tenv.data4test_dir}/test_basic_task/Taskfile.yml" + + def taskfile_text = new File(taskfile_path).text + + + + def yaml_map = api.text_yaml_to_map(taskfile_text) + def the_task = yaml_map.tasks['top-level'] + def stages = api.stage_with_steps_from_task( + taskmap: the_task, + taskname: 'top-level' + ) + + def actual = api.ast_holder.root.stages + println api.text_yaml_from_map(actual) + + def jenflow_path ="${tenv.data4test_dir}/test_basic_task/Taskfile.jenflow.yml" + def jenflow_text = new File(jenflow_path).text + def flow_full = api.text_yaml_to_map(jenflow_text) + def flow_map = flow_full.entries.stage_with_steps_from_task + def expected = flow_map.stages + + + assert actual == expected +} + + +return this \ No newline at end of file diff --git a/tasks/e2e.tasks.yml b/tasks/e2e.tasks.yml new file mode 100644 index 0000000..7b1e7ad --- /dev/null +++ b/tasks/e2e.tasks.yml @@ -0,0 +1,51 @@ +version: '3' + +tasks: + test-plain-groovy-in-docker: + desc: _ + dir: dockers/jendev + cmds: + - docker run --rm -v $(pwd):/repo --workdir /repo jupconda_img task -d src test + + test-plain-groovy-in-jup: + desc: _ + dir: dockers/jendev + cmds: + - docker-compose exec jupcondadev.samexample.com task -d ./repo/src test + + test-new-ci-bootstrap: + desc: _ + dir: dockers/jendev + cmds: + - sleep 15 + - docker-compose exec jupcondadev.samexample.com ./repo/jenpy/jenpy_cli.py test_bootstrap + + test-all: + desc: _ + deps: + - task: test-new-ci-bootstrap + - task: test-plain-groovy-in-jup + cmds: + - echo ok + + test-all-clean: + desc: _ + cmds: + - task -d dockers/jendev renew + - task: test-all + # - task -d dockers/jendev stop + + resolve-no-cache: + desc: _ + cmds: + - task jenconda:docker:build:jenconda + + ci-flow: + desc: _ + cmds: + - task: resolve-no-cache + - task: test-all-clean + - task -d dockers/jendev stop + +# https://stackoverflow.com/questions/2385553/how-can-i-generate-an-html-report-for-junit-results/2969305 +# https://gitlab.com/inorton/junit2html diff --git a/tests/complex/conftest.py b/tests/complex/conftest.py new file mode 100644 index 0000000..bdcc8b7 --- /dev/null +++ b/tests/complex/conftest.py @@ -0,0 +1,47 @@ +# content of conftest.py +import pytest + + +def pytest_collect_file(parent, path): + if path.ext == ".yaml" and path.basename.startswith("test"): + return YamlFile.from_parent(parent, fspath=path) + + +class YamlFile(pytest.File): + def collect(self): + # We need a yaml parser, e.g. PyYAML. + import yaml + + raw = yaml.safe_load(self.fspath.open()) + for name, spec in sorted(raw.items()): + yield YamlItem.from_parent(self, name=name, spec=spec) + + +class YamlItem(pytest.Item): + def __init__(self, name, parent, spec): + super().__init__(name, parent) + self.spec = spec + + def runtest(self): + for name, value in sorted(self.spec.items()): + # Some custom test execution (dumb example follows). + if name != value: + raise YamlException(self, name, value) + + def repr_failure(self, excinfo): + """Called when self.runtest() raises an exception.""" + if isinstance(excinfo.value, YamlException): + return "\n".join( + [ + "usecase execution failed", + " spec failed: {1!r}: {2!r}".format(*excinfo.value.args), + " no further details known at this point.", + ] + ) + + def reportinfo(self): + return self.fspath, 0, f"usecase: {self.name}" + + +class YamlException(Exception): + """Custom exception for error reporting.""" diff --git a/tests/complex/test_simple.yaml b/tests/complex/test_simple.yaml new file mode 100644 index 0000000..8e3e7a4 --- /dev/null +++ b/tests/complex/test_simple.yaml @@ -0,0 +1,7 @@ +# test_simple.yaml +ok: + sub1: sub1 + +hello: + world: world + some: other diff --git a/tests/test_things.py b/tests/test_things.py new file mode 100644 index 0000000..8714273 --- /dev/null +++ b/tests/test_things.py @@ -0,0 +1,2 @@ +def test_first_me(): + print("sdf") diff --git a/tools/pyinst.tasks.yml b/tools/pyinst.tasks.yml new file mode 100644 index 0000000..2b9e055 --- /dev/null +++ b/tools/pyinst.tasks.yml @@ -0,0 +1,138 @@ +#!/usr/local/bin/task --taskfile +# https://taskfile.dev + +version: '3' + +vars: + GREETING: Hello, World! + + repo_root_dir: "$(yq read _version/paths_config.yml repo_root_dir)" + proj_root_dir: "$(yq read _version/paths_config.yml proj_root_dir)" + proj_deps_dir: "$(yq read _version/paths_config.yml proj_deps_dir)" + + MX_VERSION_PREFIX__TXT: "{{.repo_root_dir}}/version/version_prefix.txt" + MX_PROJECT_PREFIX__TXT: "{{.repo_root_dir}}/version/project_prefix.txt" + MX_PROJECT_PREFIX: "$(cat ../version/project_prefix.txt)" + VENV_NAME: "{{.MX_PROJECT_PREFIX}}-pyinst37" + + DEPS_DIR: "{{.repo_root_dir}}/{{.proj_deps_dir}}" + + DEPS_CONDA_BASE: "$(cat {{.DEPS_DIR}}/conda.base.deps.list.txt | tr '\n' ' ')" + DEPS_CONDA_RUN: "$(cat {{.DEPS_DIR}}/conda.run.deps.list.txt | tr '\n' ' ')" + # yq read - {{.YQ_PATH}} --tojson | jq '.[]' | tr '\n' ' ' + # DEPS_PIP: "$(cat {{.DEPS_DIR}}/pip.deps.list.txt | tr '\n' ' ')" + DEPS_PIP_PATH: "{{.DEPS_DIR}}/pip.deps.list.txt" + + tools_json: "$(yq read {{.repo_root_dir}}/version/tools_settings.yml tool --tojson)" + conda_pm: "$(echo '{{.tools_json}}' | jq '.conda_pm' -r)" + conda_ac: "$(echo '{{.tools_json}}' | jq '.conda_ac' -r)" + + distpath: __localbuild__/pyinstaller/dist + workpath: __localbuild__/pyinstaller/build + exe_name: "$(yq read _version/pyinstaller_config.json exe_name)" + + # venv_run: "{" + +expected: + MX_VERSION_PREFIX__TXT: path + DEPS_DIR: path + +tasks: + + default: + cmds: + - task: info + + info: + desc: show info + cmds: + - echo "DEPS_CONDA_BASE={{.DEPS_CONDA_BASE}}" + - echo "DEPS_PIP={{.DEPS_PIP}}" + - echo "tools_json={{.tools_json}}" + - echo "conda_pm={{.conda_pm}}" + - echo for activate. + - echo conda activate {{.VENV_NAME}} + silent: true + + venv-create: + desc: _ + cmds: + - "{{.conda_pm}} create -y -n {{.VENV_NAME}}" + - task: venv-update-conda-base + status: + - "{{.conda_pm}} run -n {{.VENV_NAME}} conda" + + venv-update-conda-base: + cmds: + - "{{.conda_pm}} install -y -n {{.VENV_NAME}} {{.DEPS_CONDA_BASE}}" + # status: + # - conda_pm list -n {{.VENV_NAME}} + + venv-update-all: + desc: _ + cmds: + - task: venv-update-conda-run + - task: venv-update-pyinstaller-pydantic + - task: venv-update-pip + + venv-ensure: + desc: make sure venv is present + cmds: + - task: venv-create + - task: venv-update-all + + + venv-update-conda-run: + cmds: + - "{{.conda_pm}} install -y -n {{.VENV_NAME}} {{.DEPS_CONDA_RUN}}" + # status: + # - conda_pm list -n {{.VENV_NAME}} + + venv-update-pyinstaller-pydantic: + cmds: + - echo {{.TASK}} + - | + export SKIP_CYTHON=1 \ + && {{.conda_pm}} run -n {{.VENV_NAME}} \ + pip install git+https://github.com/samuelcolvin/pydantic.git@v1.6.1 + + venv-update-pip: + cmds: + - | + {{.conda_pm}} run -n {{.VENV_NAME}} pip install -r {{.DEPS_PIP_PATH}} + # status: + # - conda_pm run -n {{.VENV_NAME}} pip freeze + + test-exe-help: "{{.distpath}}/{{.exe_name}} --help | tee /dev/null" + + rebuild: + - rm -rf __localbuild__/pyinstaller + - task: build + - task: test-exe-help + + cleanbuild: + desc: validate deps before build + cmds: + - task: venv-ensure + - task: rebuild + + build: + desc: Demo of build with pyinstaller2 + target: "MX_PROJECT_PREFIX" + cmds: + - mkdir -p __localbuild__/pyinstaller + # - | + # {{.VENV_RUN}} {{.MXPACK_EXE}} pyinstaller-build info-dump + - | + bash --login -c ' + conda activate {{.VENV_NAME}}; + pyinstaller pyinstaller.spec \ + --log-level=INFO \ + --onefile \ + --distpath {{.distpath}} \ + --workpath {{.workpath}} + ' + # - | + # {{.VENV_RUN}} {{.MXPACK_EXE}} pyinstaller-build post-dump + + diff --git a/tools/venv-ensure b/tools/venv-ensure new file mode 100755 index 0000000..13008e1 --- /dev/null +++ b/tools/venv-ensure @@ -0,0 +1,3 @@ +#!/bin/bash + +echo $0 \ No newline at end of file diff --git a/tools/venv.yml b/tools/venv.yml new file mode 100755 index 0000000..fb225c6 --- /dev/null +++ b/tools/venv.yml @@ -0,0 +1,11 @@ +tasks: + ensure: + - mamba install + create: + - mamba creates + update: + - mamba update + activate: + - echo conda activate + + diff --git a/vars/jakelib.groovy b/vars/jakelib.groovy new file mode 100644 index 0000000..30c6dd3 --- /dev/null +++ b/vars/jakelib.groovy @@ -0,0 +1,84 @@ +// --- docs --- + +// https://itnext.io/jenkins-shared-libraries-part-1-5ba3d072536a +// https://tomd.xyz/jenkins-shared-library/ +// https://www.jenkins.io/doc/book/pipeline/shared-libraries/ +// https://groovy-playground.appspot.com/ +// https://groovy-lang.gitlab.io/101-scripts/docker/basico-en.html +// https://github.com/groovy/docker-groovy + + +// --- yaml sample --- +// @Grab('org.yaml:snakeyaml:1.17') +// import org.yaml.snakeyaml.Yaml + +// --- private --- +import org.jenkinsci.plugins.pipeline.utility.steps.shaded.org.yaml.snakeyaml.Yaml + +@NonCPS +def read_yaml(text) { + // @url_from=http://jenkins-ci.361315.n4.nabble.com/SnakeYaml-or-why-is-writeYaml-Limited-td4976510.html + def yaml_parser = new Yaml() + def config = yaml_parser.load(text) + return config +} + +def remove_quotes(txt){ + if (txt[0] == txt[-1]){ + // println "same" + if (txt[0] == "'" || txt[0] == '"') { + // println txt.substring(1,txt.length()-1) + return txt.substring(1,txt.length()-1) + } + } + return txt +} + +def parse_groups(env_file_conf, line){ + def full_var_name = line.takeWhile { it != '=' } + def var_content_raw = line.substring(full_var_name.length()+1) + def var_content_clean = remove_quotes(var_content_raw) + + def group_name = full_var_name.takeWhile { it != '_' } + def pack_name_long = full_var_name.substring(group_name.length()+1) + println ( [group_name, pack_name_long] ) + + def pack_name = pack_name_long.split('__')[0] + def var_name_clean = pack_name_long.split('__')[1] + + println ( [pack_name, var_name_clean] ) + + env_file_conf.full << ["${full_var_name}": "${var_content_clean}"] + env_file_conf.vars << ["${var_name_clean}": "${var_content_clean}"] + if(! env_file_conf.packs[pack_name]){ + env_file_conf.packs[pack_name] = [:] + } + env_file_conf.packs[pack_name] << ["${var_name_clean}": "${var_content_clean}"] + if(! env_file_conf.groups){ + env_file_conf.groups[group_name] = [:] + } + env_file_conf.groups[group_name] << ["${pack_name_long}": "${var_content_clean}"] +} + +// --- public --- + +def parse_env_file_text(env_file_conf, multiline_text){ + def lines = multiline_text.readLines() + def mxpack_lines = lines.collectMany { + (it.trim() =~ /^MXPACK_.*/) + ? [it.trim()] + : [] + } + def _env_file_conf = [ + full: [:], + groups: [ + MXPACK: [:] + ], + vars: [:], + packs: [:] + ] + mxpack_lines.each { it -> parse_groups(_env_file_conf, it)} + env_file_conf << _env_file_conf + return env_file_conf +} + diff --git a/vars/jen.groovy b/vars/jen.groovy index 53896d9..be214d5 100644 --- a/vars/jen.groovy +++ b/vars/jen.groovy @@ -7,7 +7,7 @@ def set_build_name(_currentBuild, scmvars, more){ _currentBuild.displayName = _currentBuild.displayName + "-" + [ scmvars.GIT_BRANCH, scmvars.GIT_COMMIT.substring(0, 7), - more].join('#') + more].join('-#') } @@ -25,50 +25,75 @@ def add_stages(jg, filename, prefix){ def step_stages_from_tasks(jg, wd, filename, root_job){ // create piple stages from task commands + jg.tstages = jg.tstages ?: [:] def taskfile = add_stages(jg, filename, '') echo 'create stages from tasks' stage_names = [] jg.cmds = taskfile['tasks'][root_job]['cmds'] for (int i = 0; i < jg.cmds.size(); i++) { def _cmd = jg.cmds[i] - def name = "" - def cmd = "" + def name = _cmd + def org_cmd = _cmd def deps = [] def dp_cmds = [] + def task_dir = '.' + // echo "FFF ${_cmd}" + // assume task:taskname syntax try { - name = _cmd['task'] - cmd = "task ${name}" + def try_some_name = _cmd['task'] + // cmd = "task ${name}" def content = taskfile['tasks'][name] if (content.deps){ + task_dir = content.dir ?: '.' deps = content.deps dp_cmds = content.cmds } } + // try task taskname syntax catch(Exception e) { - name = _cmd - cmd = _cmd - deps = [] + try { + def arr = _cmd.split() + if (arr[0]=='task' && arr.size()==2){ + name = arr[1] + def content = taskfile['tasks'][name] + if (content.deps){ + deps = content.deps + dp_cmds = content.cmds + } + } + } + catch(Exception e2) { + } } + if (deps) { - stage("deps:$name"){ + jinEchoMark("creating parallel part from ${org_cmd} => ${deps}") + // create parallel stages from cmds in deps task if exists + stage("$name:deps"){ dir(wd){ def actors = jinMakeParallel(deps, {dname -> sh "task $dname"}) parallel actors } } - stage(name){ - dir(wd){ - for (int j = 0; j < dp_cmds.size(); j++) { - def s_cmd = dp_cmds[j] - sh s_cmd - } + // create stage from cmds in deps task if exists + if (dp_cmds){ + jinEchoMark("creating sequential part from ${org_cmd}") + stage("$name:cmds"){ + dir(wd){ + dir(task_dir){ + for (int j = 0; j < dp_cmds.size(); j++) { + def s_cmd = dp_cmds[j] + sh s_cmd + } + }} } } } else { + jinEchoMark("creating original from ${org_cmd}") stage(name){ dir(wd){ - sh cmd + sh org_cmd } } } diff --git a/vars/jen_flow.groovy b/vars/jen_flow.groovy new file mode 100644 index 0000000..2fbbcb6 --- /dev/null +++ b/vars/jen_flow.groovy @@ -0,0 +1,111 @@ +// vars/jen_flow.groovy +import org.jenkinsci.plugins.pipeline.utility.steps.shaded.org.yaml.snakeyaml.Yaml + + +def __get_real_cfg(config){ + assert config.kws + + println "@@act=info-block title=agent_env_settings_yaml" + println config.agent_env_settings_yaml + + println "@@act=init kind=read title='${config.agent_env_settings_yaml}'" + def agent_env_settings_text = readTrusted(config.agent_env_settings_yaml) + println "@@act=over kind=read title='${config.agent_env_settings_yaml}'" + println agent_env_settings_text + + def yaml_parser = new Yaml() + def parsed = yaml_parser.load(agent_env_settings_text) + + def actual_type = config.pipe_type ?: parsed.default + println "@@act=over step=resolve-config follow=mutli-line-block" + println actual_type + + def real_cfg = parsed.options[actual_type] + + println "@@act=over step=resolve-pipe_type follow=mutli-line-block" + println real_cfg.pipe_type + return real_cfg +} + +def call(Map config=[:], Closure body) { + // https://www.jenkins.io/blog/2020/10/21/a-sustainable-pattern-with-shared-library/ + // https://blog.ippon.tech/setting-up-a-shared-library-and-seed-job-in-jenkins-part-2/ + // https://kimsereylam.com/jenkins/2019/12/27/jenkins-shared-libraries.html + def real_cfg = __get_real_cfg(config) + if (real_cfg['pipe_type'] == 'native'){ + ///// --- schema.by.sample ---- + // native: + // pipe_type: native + // node_label: algo-ci-generic-deb8-x86_64 + node(real_cfg['node_label']) { + properties([ + disableConcurrentBuilds() + ]) + + timestamps { + stage_fetch(config.kws, config['kws_config_file']) + body() + }} + } + if (real_cfg.pipe_type == 'k8s'){ + println real_cfg + ///// --- schema.by.sample ---- + // k8s: + // pipe_type: k8s + // pod_template_yaml: _infra/jenlib_lib/cicd/mkdocs.deliver.pipe.pod.tmpl.yaml + // cloud: k8s-objx-prod-28102020 + // container: mxcc-prod + // # FIX: change to objx + // ssh-agent: yairda-ssh-gitlab + + podTemplate( + cloud: real_cfg['cloud'], + yaml: readTrusted(real_cfg['pod_template_yaml']) + ){ + node(POD_LABEL) { timestamps { + container(real_cfg['container']){ + sshagent([real_cfg['ssh-agent']]) { + stage_fetch(config.kws, config['kws_config_file']) + body() + }} + }} + } + } +} + +def stage_fetch(kws, kws_config_file){ + stage('fetch') { + // k8s support ssh keys mapping + sh "SSH_AUTH_SOCK=${env.SSH_AUTH_SOCK} ssh-add -l || echo 'not need on k8s'" + + if (kws.params.delete_work_dir) { + echo "@jenkins: deleting working dir" + deleteDir() + } + + echo 'Fetch source' + + def scmvars = checkout scm + + echo 'Fetched source' + + def pipe_config = readYaml file: kws_config_file + kws << pipe_config + + dir(kws.const.subpath) { + def version_prefix = readFile("version/version_prefix.txt") + def project_prefix_txt = "version/project_prefix.txt" + def project_prefix = null + if (fileExists(project_prefix_txt)){ + project_prefix = readFile(project_prefix_txt) + } + else { + project_prefix = readFile('version/project_name.txt') + } + def build_name_suffix = "${kws.const.build_type}:${project_prefix}:${version_prefix}" + jen.set_build_name(currentBuild, scmvars, build_name_suffix) + kws['scmvars'] = scmvars + jen.desc_from_commits(currentBuild, kws) + } + } +} diff --git a/vars/jen_flow_native.groovy b/vars/jen_flow_native.groovy new file mode 100644 index 0000000..2119943 --- /dev/null +++ b/vars/jen_flow_native.groovy @@ -0,0 +1,48 @@ +def call(Map config=[:], Closure body) { + println config + def node_label = config.node_label; + def kws_config_file = config.kws_config_file + println "kws_config_file=${kws_config_file}" + def kws = config.kws + kws.params = kws.params ?: [:] + + + node(node_label) { + properties([ + disableConcurrentBuilds() + ]) + + timestamps { + + stage('fetch') { + if (kws.params.delete_work_dir) { + echo "@jenkins: deleting working dir" + deleteDir() + } + + echo 'Fetch source' + def scmvars = checkout scm + def pipe_config = readYaml file: kws_config_file + kws << pipe_config + + dir(kws.const.subpath) { + def version_prefix = readFile("version/version_prefix.txt") + def project_prefix_txt = "version/project_prefix.txt" + if (fileExists(project_prefix_txt)){ + def project_prefix = readFile(project_prefix_txt) + } + else { + def project_prefix = readFile('version/project_name.txt') + } + def build_name_suffix = "${kws.const.build_type}:${project_prefix}:${version_prefix}" + jen.set_build_name(currentBuild, scmvars, build_name_suffix) + kws['scmvars'] = scmvars + jen.desc_from_commits(currentBuild, kws) + } + } + + body() + + }} + +} \ No newline at end of file diff --git a/vars/jen_pipe.groovy b/vars/jen_pipe.groovy new file mode 100644 index 0000000..c70bab8 --- /dev/null +++ b/vars/jen_pipe.groovy @@ -0,0 +1,117 @@ +// vars/jen_flow.groovy +import org.jenkinsci.plugins.pipeline.utility.steps.shaded.org.yaml.snakeyaml.Yaml + + +def __get_real_cfg(config_agent_env_settings_yaml, pipe_type=null){ + + println "@@act=info-block title=agent_env_settings_yaml" + println config_agent_env_settings_yaml + + println "@@act=init kind=read title='${config_agent_env_settings_yaml}'" + def agent_env_settings_text = readTrusted(config_agent_env_settings_yaml) + println "@@act=over kind=read title='${config_agent_env_settings_yaml}'" + println agent_env_settings_text + + def yaml_parser = new Yaml() + def parsed_file = yaml_parser.load(agent_env_settings_text) + def spec_agents = parsed_file.spec.agents + + def actual_type = pipe_type ?: spec_agents.default + println "@@act=over step=resolve-config follow=mutli-line-block" + println actual_type + + def real_cfg = spec_agents.options[actual_type] + + println "@@act=over step=resolve-pipe_type follow=mutli-line-block" + println real_cfg.pipe_type + return real_cfg +} + +def call(Map config=[:], Closure body) { + // https://www.jenkins.io/blog/2020/10/21/a-sustainable-pattern-with-shared-library/ + // https://blog.ippon.tech/setting-up-a-shared-library-and-seed-job-in-jenkins-part-2/ + // https://kimsereylam.com/jenkins/2019/12/27/jenkins-shared-libraries.html + def jenlib_pipe_conf_yml = config.kws._layout.jenlib_pipe_conf_yml + def real_cfg = __get_real_cfg(jenlib_pipe_conf_yml) + if (real_cfg['pipe_type'] == 'native'){ + ///// --- schema.by.sample ---- + // native: + // pipe_type: native + // node_label: algo-ci-generic-deb8-x86_64 + node(real_cfg['node_label']) { + properties([ + disableConcurrentBuilds() + ]) + + timestamps { + ansiColor('xterm') { + stage_fetch(config.kws, jenlib_pipe_conf_yml) + body() + }} + } + } + if (real_cfg.pipe_type == 'k8s'){ + println real_cfg + ///// --- schema.by.sample ---- + // k8s: + // pipe_type: k8s + // pod_template_yaml: _infra/jenlib_lib/cicd/mkdocs.deliver.pipe.pod.tmpl.yaml + // cloud: k8s-objx-prod-28102020 + // container: mxcc-prod + // # FIX: change to objx + // ssh-agent: yairda-ssh-gitlab + + podTemplate( + cloud: real_cfg['cloud'], + yaml: readTrusted(real_cfg['pod_template_yaml']) + ){ + node(POD_LABEL) { + timestamps { + ansiColor('xterm') { + container(real_cfg['container']){ + sshagent([real_cfg['ssh-agent']]) { + stage_fetch(config.kws, jenlib_pipe_conf_yml) + body() + }} + }} + } + } + } +} + +def stage_fetch(kws, kws_config_file){ + stage('fetch') { + // k8s support ssh keys mapping + sh "SSH_AUTH_SOCK=${env.SSH_AUTH_SOCK} ssh-add -l" + + if (kws.params.delete_work_dir) { + echo "@jenkins: deleting working dir" + deleteDir() + } + + echo 'Fetch source' + + def scmvars = checkout scm + + echo 'Fetched source' + + def pipe_config_cont = readYaml file: kws_config_file + kws << pipe_config_cont.spec.kws + + dir(kws.const.subpath) { + def version_prefix = readFile("version/version_prefix.txt") + def project_prefix_txt = "version/project_prefix.txt" + def project_prefix = null + if (fileExists(project_prefix_txt)){ + project_prefix = readFile(project_prefix_txt) + } + else { + project_prefix = readFile('version/project_name.txt') + } + def build_name_suffix = "${kws.const.build_type}:${project_prefix}:${version_prefix}" + jen.set_build_name(currentBuild, scmvars, build_name_suffix) + kws['scmvars'] = scmvars + jen.desc_from_commits(currentBuild, kws) + } + } +} diff --git a/vars/jenlib_consts.groovy b/vars/jenlib_consts.groovy new file mode 100644 index 0000000..27fdce0 --- /dev/null +++ b/vars/jenlib_consts.groovy @@ -0,0 +1,4 @@ +class jenlib_consts { + final String version__version_prefix_txt = "version/jenlib_config.yml" + +} diff --git a/vars/jens.groovy b/vars/jens.groovy new file mode 100644 index 0000000..be214d5 --- /dev/null +++ b/vars/jens.groovy @@ -0,0 +1,120 @@ +#!/usr/bin/env groovy + +// jen.groovy + +def set_build_name(_currentBuild, scmvars, more){ + // set build name from scmvars + _currentBuild.displayName = _currentBuild.displayName + "-" + [ + scmvars.GIT_BRANCH, + scmvars.GIT_COMMIT.substring(0, 7), + more].join('-#') +} + + +def add_stages(jg, filename, prefix){ + // echo "Load tasks from ${filename}" + // filename='Taskfile.yml' + def taskfile = readYaml file: filename + for ( e in taskfile['tasks'] ) { + jg.tstages[prefix + e.key] = e.value + // echo "adding ${prefix}${e.key}" + } + echo "Loaded tasks from ${filename}" + return taskfile +} + +def step_stages_from_tasks(jg, wd, filename, root_job){ + // create piple stages from task commands + jg.tstages = jg.tstages ?: [:] + def taskfile = add_stages(jg, filename, '') + echo 'create stages from tasks' + stage_names = [] + jg.cmds = taskfile['tasks'][root_job]['cmds'] + for (int i = 0; i < jg.cmds.size(); i++) { + def _cmd = jg.cmds[i] + def name = _cmd + def org_cmd = _cmd + def deps = [] + def dp_cmds = [] + def task_dir = '.' + // echo "FFF ${_cmd}" + // assume task:taskname syntax + try { + def try_some_name = _cmd['task'] + // cmd = "task ${name}" + def content = taskfile['tasks'][name] + if (content.deps){ + task_dir = content.dir ?: '.' + deps = content.deps + dp_cmds = content.cmds + } + } + // try task taskname syntax + catch(Exception e) { + try { + def arr = _cmd.split() + if (arr[0]=='task' && arr.size()==2){ + name = arr[1] + def content = taskfile['tasks'][name] + if (content.deps){ + deps = content.deps + dp_cmds = content.cmds + } + } + } + catch(Exception e2) { + + } + } + + if (deps) { + jinEchoMark("creating parallel part from ${org_cmd} => ${deps}") + // create parallel stages from cmds in deps task if exists + stage("$name:deps"){ + dir(wd){ + def actors = jinMakeParallel(deps, {dname -> sh "task $dname"}) + parallel actors + } + } + // create stage from cmds in deps task if exists + if (dp_cmds){ + jinEchoMark("creating sequential part from ${org_cmd}") + stage("$name:cmds"){ + dir(wd){ + dir(task_dir){ + for (int j = 0; j < dp_cmds.size(); j++) { + def s_cmd = dp_cmds[j] + sh s_cmd + } + }} + } + } + } else { + jinEchoMark("creating original from ${org_cmd}") + stage(name){ + dir(wd){ + sh org_cmd + } + } + } + } +} + +def desc_from_commits(currentBuild, jg){ + def changeLogSets = currentBuild.changeSets + for (int i = 0; i < changeLogSets.size(); i++) { + def entries = changeLogSets[i].items + for (int j = 0; j < entries.length; j++) { + def entry = entries[j] + echo "${entry.commitId} by ${entry.author} on ${new Date(entry.timestamp)}: ${entry.msg}" + jg.commits.add("${entry.author}: ${entry.msg}".toString()) + + def files = new ArrayList(entry.affectedFiles) + for (int k = 0; k < files.size(); k++) { + def file = files[k] + echo " ${file.editType.name} ${file.path}" + } + } + } + currentBuild.description = jg.commits.join(', ') +} diff --git a/version/descriptor.yml b/version/descriptor.yml new file mode 100644 index 0000000..087f16f --- /dev/null +++ b/version/descriptor.yml @@ -0,0 +1,38 @@ +apiVersion: projects/v1 +kind: ProjectDescriptor + +metadata: + name: jenlib +spec: + layout: + required: + descriptor: + path: version/descriptor.yml + desc: | + this file gives logical names real values + + version_prefix: + path: version/version_prefix.txt + desc: | + first character of version + usually 3 first parts of semver + example: 0.3.41 + + project_prefix: + path: version/project_prefix.txt + desc: | + project public name + + git_tag_prefix: + path: version/git_tag_prefix.txt + desc: | + prefix for git tags + existed for monorepos with multiple projects + + optional: + + mkdocs_settings: + path: mkdocs.yml + desc: | + settings for mkdocs build + diff --git a/version/git_tag_prefix.txt b/version/git_tag_prefix.txt new file mode 100644 index 0000000..e69de29 diff --git a/version/project_prefix.txt b/version/project_prefix.txt new file mode 100644 index 0000000..fbac7cd --- /dev/null +++ b/version/project_prefix.txt @@ -0,0 +1 @@ +jenpack \ No newline at end of file diff --git a/version/recepies/dockerlib.yml b/version/recepies/dockerlib.yml new file mode 100644 index 0000000..e69de29 diff --git a/version/recepies/jenlib.yml b/version/recepies/jenlib.yml new file mode 100644 index 0000000..e69de29 diff --git a/version/tools_settings.yml b/version/tools_settings.yml new file mode 100644 index 0000000..3e2a081 --- /dev/null +++ b/version/tools_settings.yml @@ -0,0 +1,3 @@ +tool: + conda_pm: conda + conda_ac: conda activate diff --git a/version/version_prefix.txt b/version/version_prefix.txt new file mode 100644 index 0000000..da2ac9c --- /dev/null +++ b/version/version_prefix.txt @@ -0,0 +1 @@ +0.7.5 \ No newline at end of file