From 908e60a7b78ea3835a1423d23393ee209494ff56 Mon Sep 17 00:00:00 2001 From: Andrew Weiss Date: Tue, 13 Aug 2019 11:34:38 -0400 Subject: [PATCH 1/2] test actions workflow --- .circleci/config.yml | 490 ------------------ .github/workflows/main.yml | 529 ++++++++++++++++++++ build/ci-cd/copy-and-convert-content.sh | 4 +- build/ci-cd/generate-model-documentation.sh | 4 +- build/ci-cd/validate-content.sh | 4 +- build/ci-cd/validate-round-trips.sh | 6 +- 6 files changed, 538 insertions(+), 499 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/workflows/main.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 1587b1c0f1..0000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,490 +0,0 @@ -version: 2.1 # use CircleCI 2.1 -references: - common_environment: &common_environment - environment: - SAXON_VERSION: 9.9.0-1 - JSON_CLI_VERSION: 0.0.1-SNAPSHOT - CICD_DIR: git-oscal/build/ci-cd - OSCAL_REPO_DIR: git-oscal - OSCAL_BUILD_DIR: build_artifacts - OSCAL_TOOLS_DIR: oscal_tools - SCHEMATRON_HOME: git-schematron - TERM: xterm - working_directory: ~/oscal - attach_build_workspace: &attach_build_workspace - attach_workspace: - at: build_artifacts - checkout: &checkout - checkout: - path: git-oscal - publish_artifacts: &publish_artifacts - store_artifacts: - path: build_artifacts - restore_gem_cache: &restore_gem_cache - restore_cache: - keys: - - v2-gem-cache-{{ arch }}-{{ checksum "git-oscal/docs/Gemfile.lock" }} - save_gem_cache: &save_gem_cache - save_cache: - key: v2-gem-cache-{{ arch }}-{{ checksum "git-oscal/docs/Gemfile.lock" }} - paths: - - ~/jekyll-bundle -commands: - install-schematron: - description: "Install the schematron skeleton" - steps: - - run: - name: Get schematron skeleton - command: | - git clone --depth 1 --no-checkout https://github.com/Schematron/schematron.git "$SCHEMATRON_HOME" - cd "$SCHEMATRON_HOME" - git checkout master -- trunk/schematron/code - install-maven-dependencies: - description: "Install Maven Dependencies" - steps: - - run: - name: Get OSCAL Tools - command: | - git clone --depth 1 https://github.com/usnistgov/oscal-tools.git "${OSCAL_TOOLS_DIR}" - - run: - name: Generate cache checksum - command: | - # Ensure we cache the necessary dependencies - cat "${OSCAL_TOOLS_DIR}/json-cli/pom.xml" > .maven-cache-checksum - echo "$SAXON_VERSION" >> .maven-cache-checksum - - restore_cache: - keys: - - mvn-m2-cache-V2-{{ checksum ".maven-cache-checksum" }} - - mvn-m2-cache-V2 - - run: - name: Get Saxon-HE - command: | - mvn org.apache.maven.plugins:maven-dependency-plugin:2.10:get -DartifactId=Saxon-HE -DgroupId=net.sf.saxon -Dversion=$SAXON_VERSION - - run: - name: Build JSON Validation CLI - command: | - # update maven version - cd /opt - sudo wget https://www-us.apache.org/dist/maven/maven-3/3.6.0/binaries/apache-maven-3.6.0-bin.tar.gz - sudo tar -xvzf apache-maven-3.6.0-bin.tar.gz - sudo mv apache-maven-3.6.0 maven - export M2_HOME=/opt/maven - export PATH=${M2_HOME}/bin:${PATH} - cd - - # now build - cd "${OSCAL_TOOLS_DIR}/json-cli" - /opt/maven/bin/mvn dependency:go-offline - /opt/maven/bin/mvn install - - save_cache: - paths: - - ~/.m2 - key: mvn-m2-cache-V2-{{ checksum ".maven-cache-checksum" }} - install-maven: - description: "Installs Maven" - steps: - - run: - name: Get Maven - command: | - sudo apt-get install maven - install-lxml: - description: "Installs Python lxml package" - steps: - - run: - name: Get lxml - command: | - sudo pip install lxml - install-xmllint: - description: "Install xmllint" - steps: - - run: - name: Get xmllint - command: | - sudo apt-get install libxml2-utils - install-ajv: - description: "Install ajv" - steps: - - run: - name: Get ajv - command: | - sudo npm install -g ajv-cli - install-markdown-link-check: - description: "Install markdown-link-check" - steps: - - run: - name: Get markdown-link-check - command: | - sudo npm install -g markdown-link-check - install-jq: - description: "Install jq" - steps: - - run: - name: Get jsome - command: | - sudo apt-get install jq - install-prettyjson: - description: "Install prettyjson" - steps: - - run: - name: Get prettyjson - command: | - sudo npm install -g prettyjson - install-github-release: - description: "Install Github Release" - steps: - - run: - name: Get Github Release - command: | - go get github.com/aktau/github-release - install-bundler: - description: "Install bundler and html-proofer" - steps: - - run: - name: Install Bundler - command: | - cd "$OSCAL_REPO_DIR/docs" - echo gem install bundler - bundle install --path ~/jekyll-bundle - gem install html-proofer - check-skip-build: - steps: - - run: - name: Check if job should be skipped - command: | - if [[ -f "$OSCAL_BUILD_DIR/.skipbuild" ]]; then - echo "Skipping job." - circleci step halt - fi -executors: - java-executor: - docker: # run the java jobs with Docker - - image: circleci/openjdk:11-jdk-node-browsers - <<: *common_environment - ruby-node-executor: - docker: # run the ruby jobs with Docker - - image: circleci/ruby:2.4.6-node-browsers - <<: *common_environment - python-executor: - docker: #run the python jobs with Docker - - image: circleci/python:3.7.3-node-browsers - <<: *common_environment - go-executor: - docker: # run the steps with Docker - - image: circleci/golang:1.12.5 - <<: *common_environment -jobs: # a collection of - job-ci-skip-check: - executor: java-executor - steps: - - checkout - - run: - name: Check if skip build - command: | - mkdir -p "$OSCAL_BUILD_DIR/state" - COMMIT_MSG=$(git log --format=oneline -n 1 $CIRCLE_SHA1) - if [[ $COMMIT_MSG == *"[ci deploy skip]"* ]]; then - echo "[ci deploy skip] detected. Stopping." - echo "branch: $CIRCLE_BRANCH" > "$OSCAL_BUILD_DIR/.skipbuild" - else - echo "branch: $CIRCLE_BRANCH" > "$OSCAL_BUILD_DIR/.runbuild" - fi - - persist_to_workspace: - root: build_artifacts - paths: - - .runbuild - - .skipbuild - job-validate-metaschema: - executor: java-executor - steps: - - *checkout - - *attach_build_workspace - - check-skip-build - - install-xmllint - - install-maven-dependencies - - install-schematron - - run: - name: Validate metaschema instances - command: | - bash "$CICD_DIR/validate-metaschema.sh" "$OSCAL_BUILD_DIR" - - *publish_artifacts - - persist_to_workspace: - root: build_artifacts - paths: - - svrl - job-generate-schema: - executor: java-executor - steps: # a collection of executable commands - - *checkout - - *attach_build_workspace - - check-skip-build - - install-maven-dependencies - - run: - name: Generate OSCAL schemas - command: | - bash "$CICD_DIR/generate-schema.sh" "$OSCAL_BUILD_DIR" - - *publish_artifacts - - persist_to_workspace: - root: build_artifacts - paths: - - xml - - json - job-generate-converters: - executor: java-executor - steps: # a collection of executable commands - - *checkout - - *attach_build_workspace - - check-skip-build - - install-maven-dependencies - - run: - name: Generate OSCAL schemas - command: | - bash "$CICD_DIR/generate-content-converters.sh" "$OSCAL_BUILD_DIR" - - *publish_artifacts - - persist_to_workspace: - root: build_artifacts - paths: - - xml - - json - job-copy-and-convert-content: - executor: java-executor - steps: # a collection of executable commands - - *checkout - - *attach_build_workspace - - check-skip-build - - install-maven-dependencies - - install-jq - - install-prettyjson - - run: - name: Generate OSCAL converters - command: | - bash "$CICD_DIR/copy-and-convert-content.sh" "$OSCAL_BUILD_DIR" - - *publish_artifacts - - persist_to_workspace: - root: build_artifacts - paths: - - content # not needed? - job-validate-markdown: - executor: java-executor - steps: - - *checkout - - *attach_build_workspace - - check-skip-build - - install-markdown-link-check - - run: - name: Validate content instances - command: | - find "$OSCAL_REPO_DIR" -path ./docs -prune -o -name \*.md -exec markdown-link-check -q {} \; - job-validate-content: - executor: java-executor - steps: - - *checkout - - *attach_build_workspace - - check-skip-build - - install-xmllint - - install-maven-dependencies - - run: - name: Validate content instances - command: | - bash "$CICD_DIR/validate-content.sh" "$OSCAL_BUILD_DIR" - job-roundtrip-conversions: - executor: python-executor - steps: - - *checkout - - *attach_build_workspace - - check-skip-build - - install-maven - - install-maven-dependencies - - install-lxml - - install-xmllint - - run: - name: Generate round trip XML and JSON conversions - command: | - bash "${CICD_DIR}/validate-round-trips.sh" "$OSCAL_BUILD_DIR" - - *publish_artifacts - job-generate-docs: - executor: java-executor - steps: - - *checkout - - *attach_build_workspace - - check-skip-build - - install-maven-dependencies - - run: - name: Generate schema documentation - command: | - #cp -r "$OSCAL_BUILD_DIR"/* "$OSCAL_REPO_DIR" - bash -x "$CICD_DIR/generate-model-documentation.sh" "$OSCAL_BUILD_DIR" - # echo "-- fix the fact that we cannot generate directly to the build directory due to a bug" - # mv docs "$OSCAL_BUILD_DIR"/docs - - *publish_artifacts - - persist_to_workspace: - root: build_artifacts - paths: - - docs - job-deploy-artifacts: - executor: java-executor - steps: # a collection of executable commands - - *checkout - - *attach_build_workspace - - check-skip-build - - deploy: # delete and copy artifacts before deploy - name: Commit Artifacts - command: | - cd "$OSCAL_REPO_DIR" - # Remove existing generated files - git rm -r --ignore-unmatch xml/convert/*.xsl - git rm -r --ignore-unmatch xml/schema/*.xsd - git rm -r --ignore-unmatch json/convert/*.xsl - git rm -r --ignore-unmatch json/schema/*.json - git rm -r --ignore-unmatch content/**/*.xml - git rm -r --ignore-unmatch content/**/*.json - git rm -r --ignore-unmatch content/**/*.yaml - # Copy new built files to repo - cd - - cp -r "$OSCAL_BUILD_DIR"/* "$OSCAL_REPO_DIR" - cd "$OSCAL_REPO_DIR" - # add the new files - git add -f --all xml - git add -f --all json - git add -f --all content - # check for changes - echo "Changed files:" - - if ! $(git diff --exit-code --name-only HEAD~1 xml json content); then - # Only deploy if something relevant has changed - # Setup deployment - git config user.name "Deployment Bot" - git commit --allow-empty -m "Deploying content [ci deploy skip]" - # Ensure we are deploying against the latest - git fetch - git rebase origin/master - # deploy - git push - else - echo " No files changed" - fi - - *publish_artifacts - job-validate-site-content: - executor: ruby-node-executor - steps: - - *checkout - - *attach_build_workspace - - check-skip-build - - *restore_gem_cache - - install-bundler - - run: - name: Build and Validate Jekyll site - command: | - cp -vr "$OSCAL_BUILD_DIR"/* "$OSCAL_REPO_DIR" - cd "$OSCAL_REPO_DIR/docs" - bundle exec jekyll build -d _site/OSCAL - htmlproofer --assume-extension ./_site - environment: - JEKYLL_ENV: production - - *save_gem_cache - - *publish_artifacts - job-build-and-deploy-website: - executor: ruby-node-executor - steps: - - *checkout - - *attach_build_workspace - - check-skip-build - - *restore_gem_cache - - install-bundler - - run: - name: Build jekyll site - command: | - cp -vr "$OSCAL_BUILD_DIR"/* "$OSCAL_REPO_DIR" - cd "$OSCAL_REPO_DIR/docs" - bundle exec jekyll build - mkdir -p _site/.circleci - # copy the circleci config - cp ../.circleci/config.yml _site/.circleci/config.yml - environment: - JEKYLL_ENV: production - - *save_gem_cache - - run: - name: Run deploy script - command: | - cd "$OSCAL_REPO_DIR/docs" - git config user.name "Deployment Bot" - bash ./deploy.sh --push-only -v -m "Deploying website [ci deploy skip]" - - *publish_artifacts - package: - executor: go-executor - steps: - - *checkout - - install-github-release - - run: - name: Package Release - command: | - bash -x "$CICD_DIR/package-release.sh" "$OSCAL_BUILD_DIR" -workflows: - version: 2 - build: - jobs: - - package: - filters: - branches: - ignore: /.*/ - tags: - only: /v\d+.+/ - - job-ci-skip-check: - filters: - tags: - ignore: /.*/ - - job-validate-markdown: - requires: - - job-ci-skip-check - filters: - branches: - ignore: - - master - - nist-pages - - job-validate-metaschema: - requires: - - job-ci-skip-check - filters: - branches: - ignore: nist-pages - - job-generate-schema: - requires: - - job-validate-metaschema - - job-generate-converters: - requires: - - job-validate-metaschema - - job-validate-content: - requires: - - job-generate-schema - - job-roundtrip-conversions: - requires: - - job-generate-converters - - job-validate-content - - job-copy-and-convert-content: - requires: - - job-generate-converters - - job-validate-content - - job-generate-docs: - requires: - - job-validate-metaschema - - job-generate-converters - - job-validate-site-content: - requires: - - job-generate-docs - filters: - branches: - # only from PRs - ignore: - - master - - nist-pages - - job-deploy-artifacts: - requires: - - job-copy-and-convert-content - filters: - branches: - only: master - - job-build-and-deploy-website: - requires: - - job-generate-docs - - job-deploy-artifacts - filters: - branches: - only: master diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000000..e54fed8d9d --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,529 @@ +on: [push, pull_request] + +jobs: + validate-metaschema: + strategy: + matrix: + os: [ ubuntu-latest ] + saxon: [ 9.9.0-1 ] + json-cli: [ 0.0.1-SNAPSHOT ] + name: validate metaschema on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + steps: + - name: Checkout source + uses: actions/checkout@master + + - name: Install xmllint + run: | + sudo apt install libxml2-utils + + - name: Install maven-dependencies + env: + SAXON_VERSION: ${{ matrix.saxon }} + JSON_CLI_VERSION: ${{ matrix.json-cli }} + CICD_DIR: build/ci-cd + OSCAL_REPO_DIR: $GITHUB_WORKSPACE + OSCAL_BUILD_DIR: build-artifacts + OSCAL_TOOLS_DIR: oscal_tools + SCHEMATRON_HOME: git-schematron + TERM: xterm + run: | + git clone --depth 1 https://github.com/usnistgov/oscal-tools.git "${OSCAL_TOOLS_DIR}" + mvn org.apache.maven.plugins:maven-dependency-plugin:2.10:get -DartifactId=Saxon-HE -DgroupId=net.sf.saxon -Dversion=$SAXON_VERSION + # update maven version + cd /opt + sudo wget https://www-us.apache.org/dist/maven/maven-3/3.6.0/binaries/apache-maven-3.6.0-bin.tar.gz + sudo tar -xvzf apache-maven-3.6.0-bin.tar.gz + sudo mv apache-maven-3.6.0 maven + export M2_HOME=/opt/maven + export PATH=${M2_HOME}/bin:${PATH} + cd - + # now build + cd "${OSCAL_TOOLS_DIR}/json-cli" + /opt/maven/bin/mvn dependency:go-offline + /opt/maven/bin/mvn install + + - name: Install schematron + env: + SAXON_VERSION: ${{ matrix.saxon }} + JSON_CLI_VERSION: ${{ matrix.json-cli }} + CICD_DIR: build/ci-cd + OSCAL_REPO_DIR: $GITHUB_WORKSPACE + OSCAL_BUILD_DIR: build-artifacts + OSCAL_TOOLS_DIR: oscal_tools + SCHEMATRON_HOME: git-schematron + TERM: xterm + run: | + git clone --depth 1 --no-checkout https://github.com/Schematron/schematron.git "$SCHEMATRON_HOME" + cd "$SCHEMATRON_HOME" + git checkout master -- trunk/schematron/code + + - name: Validate metaschema instances + env: + SAXON_VERSION: ${{ matrix.saxon }} + JSON_CLI_VERSION: ${{ matrix.json-cli }} + CICD_DIR: build/ci-cd + OSCAL_REPO_DIR: $GITHUB_WORKSPACE + OSCAL_BUILD_DIR: build-artifacts + OSCAL_TOOLS_DIR: oscal_tools + SCHEMATRON_HOME: git-schematron + TERM: xterm + run: bash "$CICD_DIR/validate-metaschema.sh" "$OSCAL_BUILD_DIR" + + - name: Publish build-artifacts + uses: actions/upload-artifact@master + with: + name: build-artifacts + path: build-artifacts + + generate-schema: + strategy: + matrix: + os: [ ubuntu-latest ] + saxon: [ 9.9.0-1 ] + json-cli: [ 0.0.1-SNAPSHOT ] + name: generate schema on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + needs: validate-metaschema + steps: + - name: Checkout source + uses: actions/checkout@master + + - name: Install maven-dependencies + env: + SAXON_VERSION: ${{ matrix.saxon }} + JSON_CLI_VERSION: ${{ matrix.json-cli }} + CICD_DIR: build/ci-cd + OSCAL_REPO_DIR: $GITHUB_WORKSPACE + OSCAL_BUILD_DIR: build-artifacts + OSCAL_TOOLS_DIR: oscal_tools + SCHEMATRON_HOME: git-schematron + TERM: xterm + run: | + git clone --depth 1 https://github.com/usnistgov/oscal-tools.git "${OSCAL_TOOLS_DIR}" + mvn org.apache.maven.plugins:maven-dependency-plugin:2.10:get -DartifactId=Saxon-HE -DgroupId=net.sf.saxon -Dversion=$SAXON_VERSION + # update maven version + cd /opt + sudo wget https://www-us.apache.org/dist/maven/maven-3/3.6.0/binaries/apache-maven-3.6.0-bin.tar.gz + sudo tar -xvzf apache-maven-3.6.0-bin.tar.gz + sudo mv apache-maven-3.6.0 maven + export M2_HOME=/opt/maven + export PATH=${M2_HOME}/bin:${PATH} + cd - + # now build + cd "${OSCAL_TOOLS_DIR}/json-cli" + /opt/maven/bin/mvn dependency:go-offline + /opt/maven/bin/mvn install + + - name: General OSCAL schemas + env: + SAXON_VERSION: ${{ matrix.saxon }} + JSON_CLI_VERSION: ${{ matrix.json-cli }} + CICD_DIR: build/ci-cd + OSCAL_REPO_DIR: $GITHUB_WORKSPACE + OSCAL_BUILD_DIR: build-artifacts + OSCAL_TOOLS_DIR: oscal_tools + SCHEMATRON_HOME: git-schematron + TERM: xterm + run: bash "$CICD_DIR/generate-schema.sh" "$OSCAL_BUILD_DIR" + + - name: Publish build-artifacts + uses: actions/upload-artifact@master + with: + name: build-artifacts + path: build-artifacts + + generate-converters: + strategy: + matrix: + os: [ ubuntu-latest ] + saxon: [ 9.9.0-1 ] + json-cli: [ 0.0.1-SNAPSHOT ] + name: generate converters on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + needs: validate-metaschema + steps: + - name: Checkout source + uses: actions/checkout@master + + - name: Install maven-dependencies + env: + SAXON_VERSION: ${{ matrix.saxon }} + JSON_CLI_VERSION: ${{ matrix.json-cli }} + CICD_DIR: build/ci-cd + OSCAL_REPO_DIR: $GITHUB_WORKSPACE + OSCAL_BUILD_DIR: build-artifacts + OSCAL_TOOLS_DIR: oscal_tools + SCHEMATRON_HOME: git-schematron + TERM: xterm + run: | + git clone --depth 1 https://github.com/usnistgov/oscal-tools.git "${OSCAL_TOOLS_DIR}" + mvn org.apache.maven.plugins:maven-dependency-plugin:2.10:get -DartifactId=Saxon-HE -DgroupId=net.sf.saxon -Dversion=$SAXON_VERSION + # update maven version + cd /opt + sudo wget https://www-us.apache.org/dist/maven/maven-3/3.6.0/binaries/apache-maven-3.6.0-bin.tar.gz + sudo tar -xvzf apache-maven-3.6.0-bin.tar.gz + sudo mv apache-maven-3.6.0 maven + export M2_HOME=/opt/maven + export PATH=${M2_HOME}/bin:${PATH} + cd - + # now build + cd "${OSCAL_TOOLS_DIR}/json-cli" + /opt/maven/bin/mvn dependency:go-offline + /opt/maven/bin/mvn install + + - name: General OSCAL schemas + env: + SAXON_VERSION: ${{ matrix.saxon }} + JSON_CLI_VERSION: ${{ matrix.json-cli }} + CICD_DIR: build/ci-cd + OSCAL_REPO_DIR: $GITHUB_WORKSPACE + OSCAL_BUILD_DIR: build-artifacts + OSCAL_TOOLS_DIR: oscal_tools + SCHEMATRON_HOME: git-schematron + TERM: xterm + run: bash "$CICD_DIR/generate-content-converters.sh" "$OSCAL_BUILD_DIR" + + - name: Publish build-artifacts + uses: actions/upload-artifact@master + with: + name: build-artifacts + path: build-artifacts + + copy-and-convert-content: + strategy: + matrix: + os: [ ubuntu-latest ] + saxon: [ 9.9.0-1 ] + json-cli: [ 0.0.1-SNAPSHOT ] + name: copy and convert content on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + needs: [ generate-converters, validate-content ] + steps: + - name: Checkout source + uses: actions/checkout@master + + - name: Install maven-dependencies + env: + SAXON_VERSION: ${{ matrix.saxon }} + JSON_CLI_VERSION: ${{ matrix.json-cli }} + CICD_DIR: build/ci-cd + OSCAL_REPO_DIR: $GITHUB_WORKSPACE + OSCAL_BUILD_DIR: build-artifacts + OSCAL_TOOLS_DIR: oscal_tools + SCHEMATRON_HOME: git-schematron + TERM: xterm + run: | + git clone --depth 1 https://github.com/usnistgov/oscal-tools.git "${OSCAL_TOOLS_DIR}" + mvn org.apache.maven.plugins:maven-dependency-plugin:2.10:get -DartifactId=Saxon-HE -DgroupId=net.sf.saxon -Dversion=$SAXON_VERSION + # update maven version + cd /opt + sudo wget https://www-us.apache.org/dist/maven/maven-3/3.6.0/binaries/apache-maven-3.6.0-bin.tar.gz + sudo tar -xvzf apache-maven-3.6.0-bin.tar.gz + sudo mv apache-maven-3.6.0 maven + export M2_HOME=/opt/maven + export PATH=${M2_HOME}/bin:${PATH} + cd - + # now build + cd "${OSCAL_TOOLS_DIR}/json-cli" + /opt/maven/bin/mvn dependency:go-offline + /opt/maven/bin/mvn install + + - name: Install jq + run: | + sudo apt install jq + + - name: Install prettyjson + run: | + sudo npm install -g prettyjson + + - name: Generate OSCAL converters + env: + SAXON_VERSION: ${{ matrix.saxon }} + JSON_CLI_VERSION: ${{ matrix.json-cli }} + CICD_DIR: build/ci-cd + OSCAL_BUILD_DIR: build-artifacts + OSCAL_TOOLS_DIR: oscal_tools + SCHEMATRON_HOME: git-schematron + TERM: xterm + run: | + bash "$CICD_DIR/copy-and-convert-content.sh" "$OSCAL_BUILD_DIR" + + - name: Publish build-artifacts + uses: actions/upload-artifact@master + with: + name: build-artifacts + path: build-artifacts + + validate-markdown: + strategy: + matrix: + os: [ ubuntu-latest ] + saxon: [ 9.9.0-1 ] + json-cli: [ 0.0.1-SNAPSHOT ] + name: validate markdown on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + steps: + - name: Checkout source + uses: actions/checkout@master + + - name: Install markdown-link-check + run: | + sudo npm install -g markdown-link-check + + - name: Validate content instances + env: + SAXON_VERSION: ${{ matrix.saxon }} + JSON_CLI_VERSION: ${{ matrix.json-cli }} + CICD_DIR: build/ci-cd + OSCAL_REPO_DIR: $GITHUB_WORKSPACE + OSCAL_BUILD_DIR: build-artifacts + OSCAL_TOOLS_DIR: oscal_tools + SCHEMATRON_HOME: git-schematron + TERM: xterm + run: find . -path ./docs -prune -o -name \*.md -exec markdown-link-check -q {} \; + + validate-content: + strategy: + matrix: + os: [ ubuntu-latest ] + saxon: [ 9.9.0-1 ] + json-cli: [ 0.0.1-SNAPSHOT ] + name: validate content on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + needs: generate-schema + steps: + - name: Checkout source + uses: actions/checkout@master + + - name: Install xmllint + run: | + sudo apt install libxml2-utils + + - name: Install maven-dependencies + env: + SAXON_VERSION: ${{ matrix.saxon }} + JSON_CLI_VERSION: ${{ matrix.json-cli }} + CICD_DIR: build/ci-cd + OSCAL_REPO_DIR: $GITHUB_WORKSPACE + OSCAL_BUILD_DIR: build-artifacts + OSCAL_TOOLS_DIR: oscal_tools + SCHEMATRON_HOME: git-schematron + TERM: xterm + run: | + git clone --depth 1 https://github.com/usnistgov/oscal-tools.git "${OSCAL_TOOLS_DIR}" + mvn org.apache.maven.plugins:maven-dependency-plugin:2.10:get -DartifactId=Saxon-HE -DgroupId=net.sf.saxon -Dversion=$SAXON_VERSION + # update maven version + cd /opt + sudo wget https://www-us.apache.org/dist/maven/maven-3/3.6.0/binaries/apache-maven-3.6.0-bin.tar.gz + sudo tar -xvzf apache-maven-3.6.0-bin.tar.gz + sudo mv apache-maven-3.6.0 maven + export M2_HOME=/opt/maven + export PATH=${M2_HOME}/bin:${PATH} + cd - + # now build + cd "${OSCAL_TOOLS_DIR}/json-cli" + /opt/maven/bin/mvn dependency:go-offline + /opt/maven/bin/mvn install + + - name: Validate content instances + env: + SAXON_VERSION: ${{ matrix.saxon }} + JSON_CLI_VERSION: ${{ matrix.json-cli }} + CICD_DIR: build/ci-cd + OSCAL_REPO_DIR: $GITHUB_WORKSPACE + OSCAL_BUILD_DIR: build-artifacts + OSCAL_TOOLS_DIR: oscal_tools + SCHEMATRON_HOME: git-schematron + TERM: xterm + run: | + bash "$CICD_DIR/validate-content.sh" "$OSCAL_BUILD_DIR" + + roundtrip-conversions: + strategy: + matrix: + os: [ ubuntu-latest ] + saxon: [ 9.9.0-1 ] + json-cli: [ 0.0.1-SNAPSHOT ] + name: roundtrip conversions on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + needs: [ generate-converters, validate-content ] + steps: + - name: Checkout source + uses: actions/checkout@master + + - name: Install maven-dependencies + env: + SAXON_VERSION: ${{ matrix.saxon }} + JSON_CLI_VERSION: ${{ matrix.json-cli }} + CICD_DIR: build/ci-cd + OSCAL_REPO_DIR: $GITHUB_WORKSPACE + OSCAL_BUILD_DIR: build-artifacts + OSCAL_TOOLS_DIR: oscal_tools + SCHEMATRON_HOME: git-schematron + TERM: xterm + run: | + git clone --depth 1 https://github.com/usnistgov/oscal-tools.git "${OSCAL_TOOLS_DIR}" + mvn org.apache.maven.plugins:maven-dependency-plugin:2.10:get -DartifactId=Saxon-HE -DgroupId=net.sf.saxon -Dversion=$SAXON_VERSION + # update maven version + cd /opt + sudo wget https://www-us.apache.org/dist/maven/maven-3/3.6.0/binaries/apache-maven-3.6.0-bin.tar.gz + sudo tar -xvzf apache-maven-3.6.0-bin.tar.gz + sudo mv apache-maven-3.6.0 maven + export M2_HOME=/opt/maven + export PATH=${M2_HOME}/bin:${PATH} + cd - + # now build + cd "${OSCAL_TOOLS_DIR}/json-cli" + /opt/maven/bin/mvn dependency:go-offline + /opt/maven/bin/mvn install + + - name: Install lxml + run: | + sudo pip install lxml + + - name: Install xmllint + run: | + sudo apt install libxml2-utils + + - name: Generate round trip XML and JSON conversions + env: + SAXON_VERSION: ${{ matrix.saxon }} + JSON_CLI_VERSION: ${{ matrix.json-cli }} + CICD_DIR: build/ci-cd + OSCAL_REPO_DIR: $GITHUB_WORKSPACE + OSCAL_BUILD_DIR: build-artifacts + OSCAL_TOOLS_DIR: oscal_tools + SCHEMATRON_HOME: git-schematron + TERM: xterm + run: | + bash "$CICD_DIR/validate-round-trips.sh" "$OSCAL_BUILD_DIR" + + - name: Publish build-artifacts + uses: actions/upload-artifact@master + with: + name: build-artifacts + path: build-artifacts + + generate-docs: + strategy: + matrix: + os: [ ubuntu-latest ] + saxon: [ 9.9.0-1 ] + json-cli: [ 0.0.1-SNAPSHOT ] + name: generate docs on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + needs: [ validate-metaschema, generate-converters ] + steps: + - name: Checkout source + uses: actions/checkout@master + + - name: Install maven-dependencies + env: + SAXON_VERSION: ${{ matrix.saxon }} + JSON_CLI_VERSION: ${{ matrix.json-cli }} + CICD_DIR: build/ci-cd + OSCAL_REPO_DIR: $GITHUB_WORKSPACE + OSCAL_BUILD_DIR: build-artifacts + OSCAL_TOOLS_DIR: oscal_tools + SCHEMATRON_HOME: git-schematron + TERM: xterm + run: | + git clone --depth 1 https://github.com/usnistgov/oscal-tools.git "${OSCAL_TOOLS_DIR}" + mvn org.apache.maven.plugins:maven-dependency-plugin:2.10:get -DartifactId=Saxon-HE -DgroupId=net.sf.saxon -Dversion=$SAXON_VERSION + # update maven version + cd /opt + sudo wget https://www-us.apache.org/dist/maven/maven-3/3.6.0/binaries/apache-maven-3.6.0-bin.tar.gz + sudo tar -xvzf apache-maven-3.6.0-bin.tar.gz + sudo mv apache-maven-3.6.0 maven + export M2_HOME=/opt/maven + export PATH=${M2_HOME}/bin:${PATH} + cd - + # now build + cd "${OSCAL_TOOLS_DIR}/json-cli" + /opt/maven/bin/mvn dependency:go-offline + /opt/maven/bin/mvn install + + - name: Generate schema documentation + env: + SAXON_VERSION: ${{ matrix.saxon }} + JSON_CLI_VERSION: ${{ matrix.json-cli }} + CICD_DIR: build/ci-cd + OSCAL_REPO_DIR: $GITHUB_WORKSPACE + OSCAL_BUILD_DIR: build-artifacts + OSCAL_TOOLS_DIR: oscal_tools + SCHEMATRON_HOME: git-schematron + TERM: xterm + run: | + bash -x "$CICD_DIR/generate-model-documentation.sh" "$OSCAL_BUILD_DIR" + + - name: Publish build-artifacts + uses: actions/upload-artifact@master + with: + name: build-artifacts + path: build-artifacts + + deploy-artifacts: + name: deploy artifacts + runs-on: ubuntu-latest + needs: copy-and-convert-content + steps: + - name: Checkout source + uses: actions/checkout@master + + - name: Download artifacts + uses: actions/download-artifact@master + with: + name: build-artifacts + + - name: Comit artifacts + env: + SAXON_VERSION: ${{ matrix.saxon }} + JSON_CLI_VERSION: ${{ matrix.json-cli }} + CICD_DIR: build/ci-cd + OSCAL_REPO_DIR: $GITHUB_WORKSPACE + OSCAL_BUILD_DIR: build-artifacts + OSCAL_TOOLS_DIR: oscal_tools + SCHEMATRON_HOME: git-schematron + TERM: xterm + run: | + cd "$OSCAL_REPO_DIR" + # Remove existing generated files + git rm -r --ignore-unmatch xml/convert/*.xsl + git rm -r --ignore-unmatch xml/schema/*.xsd + git rm -r --ignore-unmatch json/convert/*.xsl + git rm -r --ignore-unmatch json/schema/*.json + git rm -r --ignore-unmatch content/**/*.xml + git rm -r --ignore-unmatch content/**/*.json + git rm -r --ignore-unmatch content/**/*.yaml + # Copy new built files to repo + cd - + cp -r "$OSCAL_BUILD_DIR"/* "$OSCAL_REPO_DIR" + cd "$OSCAL_REPO_DIR" + # add the new files + git add -f --all xml + git add -f --all json + git add -f --all content + # check for changes + echo "Changed files:" + + if ! $(git diff --exit-code --name-only HEAD~1 xml json content); then + # Only deploy if something relevant has changed + # Setup deployment + git config user.name "Deployment Bot" + git commit --allow-empty -m "Deploying content [ci deploy skip]" + # Ensure we are deploying against the latest + git fetch + git rebase origin/master + # deploy + git push + else + echo " No files changed" + fi + + - name: Publish build-artifacts + uses: actions/upload-artifact@master + with: + name: build-artifacts + path: build-artifacts + \ No newline at end of file diff --git a/build/ci-cd/copy-and-convert-content.sh b/build/ci-cd/copy-and-convert-content.sh index e435dcf460..c0b86c166f 100644 --- a/build/ci-cd/copy-and-convert-content.sh +++ b/build/ci-cd/copy-and-convert-content.sh @@ -45,7 +45,7 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do newpath="${newpath%.*}" # strip extension dest="$working_dir/${newpath}-min.${altformat}" - converter="$working_dir/$altformat/convert/oscal_${model}_${format}-to-${altformat}-converter.xsl" + converter="$PWD/$altformat/convert/oscal_${model}_${format}-to-${altformat}-converter.xsl" echo "${P_INFO}Generating ${altformat^^} file '$dest' from '$file' using converter '$converter'.${P_END}" xsl_transform "$converter" "$file" "$dest" @@ -68,7 +68,7 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do # cp "${dest}.tmp" "${dest}" # validate generated file - schema="$working_dir/json/schema/oscal_${model}_schema.json" + schema="$PWD/json/schema/oscal_${model}_schema.json" validate_json "$schema" "$dest" cmd_exitcode=$? if [ $cmd_exitcode -ne 0 ]; then diff --git a/build/ci-cd/generate-model-documentation.sh b/build/ci-cd/generate-model-documentation.sh index 56559f18cd..b4848ed6cb 100644 --- a/build/ci-cd/generate-model-documentation.sh +++ b/build/ci-cd/generate-model-documentation.sh @@ -15,7 +15,7 @@ fi echo "${P_INFO}Working in '${P_END}${working_dir}${P_INFO}'.${P_END}" # the stylesheet used to generate the documentation -stylesheet="$OSCALDIR/build/metaschema/xml/produce-and-run-either-documentor.xsl" +stylesheet="$GITHUB_WORKSPACE/build/metaschema/xml/produce-and-run-either-documentor.xsl" # the directory to generate the documentation in schema_doc_dir="${working_dir}/docs/content/documentation/schemas" @@ -44,7 +44,7 @@ while IFS="|" read path gen_schema gen_converter gen_docs || [[ -n "$path" ]]; d extension="${filename##*.}" filename="${filename%.*}" base="${filename/_metaschema/}" - converter="$working_dir/json/convert/${base}_xml-to-json-converter.xsl" + converter="$GITHUB_WORKSPACE/json/convert/${base}_xml-to-json-converter.xsl" # Make xslt paths relative to current directory metaschema_path=$(realpath --relative-to="$PWD" "$metaschema") diff --git a/build/ci-cd/validate-content.sh b/build/ci-cd/validate-content.sh index f801c9e61d..a91eaf37fd 100644 --- a/build/ci-cd/validate-content.sh +++ b/build/ci-cd/validate-content.sh @@ -35,7 +35,7 @@ while IFS="|" read path format model converttoformats || [ -n "$path" ]; do case $format in xml) - schema="$working_dir/xml/schema/oscal_${model}_schema.xsd" + schema="$PWD/xml/schema/oscal_${model}_schema.xsd" xmllint --noout --schema "$schema" "$file" cmd_exitcode=$? if [ $cmd_exitcode -ne 0 ]; then @@ -44,7 +44,7 @@ while IFS="|" read path format model converttoformats || [ -n "$path" ]; do fi ;; json) - schema="$working_dir/json/schema/oscal_${model}_schema.json" + schema="$PWD/json/schema/oscal_${model}_schema.json" validate_json "$schema" "$file" cmd_exitcode=$? if [ $cmd_exitcode -ne 0 ]; then diff --git a/build/ci-cd/validate-round-trips.sh b/build/ci-cd/validate-round-trips.sh index 0646f55954..2b18cf319b 100644 --- a/build/ci-cd/validate-round-trips.sh +++ b/build/ci-cd/validate-round-trips.sh @@ -57,7 +57,7 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do if [ "$format" == "xml" ]; then # XML -> JSON -> XML round trip testing # transformation from source XML to target JSON - converter="$working_dir/json/convert/oscal_${model}_xml-to-json-converter.xsl" + converter="$PWD/json/convert/oscal_${model}_xml-to-json-converter.xsl" to_json="$working_dir/roundtrip/${baseName}-to.json" xsl_transform "$converter" "$file" "$to_json" @@ -75,7 +75,7 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do back_to_xml="$working_dir/roundtrip/${baseName}-back-to.xml" # transformation of JSON back to XML - converter="$working_dir/xml/convert/oscal_${model}_json-to-xml-converter.xsl" + converter="$PWD/xml/convert/oscal_${model}_json-to-xml-converter.xsl" converter_path=$(realpath --relative-to="$PWD" "$converter") output_path=$(realpath --relative-to="$PWD" "$back_to_xml") @@ -96,7 +96,7 @@ while IFS="|" read path format model converttoformats || [[ -n "$path" ]]; do fi # Validate the resulting XML - schema="$working_dir/xml/schema/oscal_${model}_schema.xsd" + schema="$PWD/xml/schema/oscal_${model}_schema.xsd" xmllint --noout --schema "$schema" "$output_path" cmd_exitcode=$? if [ $cmd_exitcode -ne 0 ]; then From d3e6c70c4b59f8ea27741c078a8eedd321fedf96 Mon Sep 17 00:00:00 2001 From: Andrew Weiss Date: Tue, 13 Aug 2019 17:56:39 -0400 Subject: [PATCH 2/2] test actions workflow --- .github/workflows/main.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e54fed8d9d..9906a6b6e2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: SAXON_VERSION: ${{ matrix.saxon }} JSON_CLI_VERSION: ${{ matrix.json-cli }} CICD_DIR: build/ci-cd - OSCAL_REPO_DIR: $GITHUB_WORKSPACE + OSCAL_REPO_DIR: "$GITHUB_WORKSPACE" OSCAL_BUILD_DIR: build-artifacts OSCAL_TOOLS_DIR: oscal_tools SCHEMATRON_HOME: git-schematron @@ -48,7 +48,7 @@ jobs: SAXON_VERSION: ${{ matrix.saxon }} JSON_CLI_VERSION: ${{ matrix.json-cli }} CICD_DIR: build/ci-cd - OSCAL_REPO_DIR: $GITHUB_WORKSPACE + OSCAL_REPO_DIR: "$GITHUB_WORKSPACE" OSCAL_BUILD_DIR: build-artifacts OSCAL_TOOLS_DIR: oscal_tools SCHEMATRON_HOME: git-schematron @@ -63,7 +63,7 @@ jobs: SAXON_VERSION: ${{ matrix.saxon }} JSON_CLI_VERSION: ${{ matrix.json-cli }} CICD_DIR: build/ci-cd - OSCAL_REPO_DIR: $GITHUB_WORKSPACE + OSCAL_REPO_DIR: "$GITHUB_WORKSPACE" OSCAL_BUILD_DIR: build-artifacts OSCAL_TOOLS_DIR: oscal_tools SCHEMATRON_HOME: git-schematron @@ -94,7 +94,7 @@ jobs: SAXON_VERSION: ${{ matrix.saxon }} JSON_CLI_VERSION: ${{ matrix.json-cli }} CICD_DIR: build/ci-cd - OSCAL_REPO_DIR: $GITHUB_WORKSPACE + OSCAL_REPO_DIR: "$GITHUB_WORKSPACE" OSCAL_BUILD_DIR: build-artifacts OSCAL_TOOLS_DIR: oscal_tools SCHEMATRON_HOME: git-schematron @@ -481,13 +481,12 @@ jobs: SAXON_VERSION: ${{ matrix.saxon }} JSON_CLI_VERSION: ${{ matrix.json-cli }} CICD_DIR: build/ci-cd - OSCAL_REPO_DIR: $GITHUB_WORKSPACE OSCAL_BUILD_DIR: build-artifacts OSCAL_TOOLS_DIR: oscal_tools SCHEMATRON_HOME: git-schematron TERM: xterm run: | - cd "$OSCAL_REPO_DIR" + cd "$GITHUB_WORKSPACE" # Remove existing generated files git rm -r --ignore-unmatch xml/convert/*.xsl git rm -r --ignore-unmatch xml/schema/*.xsd @@ -498,8 +497,8 @@ jobs: git rm -r --ignore-unmatch content/**/*.yaml # Copy new built files to repo cd - - cp -r "$OSCAL_BUILD_DIR"/* "$OSCAL_REPO_DIR" - cd "$OSCAL_REPO_DIR" + cp -r "$OSCAL_BUILD_DIR"/* "$GITHUB_WORKSPACE" + cd "$GITHUB_WORKSPACE" # add the new files git add -f --all xml git add -f --all json