diff --git a/.github/workflows/gui-ci.yml b/.github/workflows/gui-ci.yml index b942f4246..c3f009531 100644 --- a/.github/workflows/gui-ci.yml +++ b/.github/workflows/gui-ci.yml @@ -4,6 +4,8 @@ # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! name: GUI CI +env: + NIGHTLIES_TO_KEEP: 20 'on': push: branches: @@ -12,6 +14,8 @@ name: GUI CI - stable pull_request: {} workflow_dispatch: {} + schedule: + - cron: 0 6 * * 2-6 jobs: info: name: Build Info @@ -41,6 +45,17 @@ jobs: - uses: actions/checkout@v1 with: clean: false + - name: Setup nightly + env: + GITHUB_TOKEN: ${{ github.token }} + run: |2- + + node ./run nightly-gen --skip-version-validation + head CHANGELOG.md + cat config.json + + if: github.event.name == 'schedule' + shell: bash - name: Read changelog info id: changelog run: |2- @@ -66,7 +81,9 @@ jobs: run: >- if [[ ${{ steps.checkCurrentReleaseTag.outputs.exists }} == true ]]; then exit 1; fi - if: github.base_ref == 'unstable' || github.base_ref == 'stable' + if: >- + github.event.name == 'schedule' || github.base_ref == 'unstable' || + github.base_ref == 'stable' - name: Get list of changed files id: changed_files run: |2- @@ -294,6 +311,17 @@ jobs: - uses: actions/checkout@v1 with: clean: false + - name: Setup nightly + env: + GITHUB_TOKEN: ${{ github.token }} + run: |2- + + node ./run nightly-gen --skip-version-validation + head CHANGELOG.md + cat config.json + + if: github.event.name == 'schedule' + shell: bash - name: Read changelog info id: changelog run: |2- @@ -438,6 +466,17 @@ jobs: uses: actions/download-artifact@v2 with: path: artifacts + - name: Setup nightly + env: + GITHUB_TOKEN: ${{ github.token }} + run: |2- + + node ./run nightly-gen --skip-version-validation + head CHANGELOG.md + cat config.json + + if: github.event.name == 'schedule' + shell: bash - name: Read changelog info id: changelog run: |2- @@ -457,7 +496,9 @@ jobs: run: >- if [[ ${{ steps.checkCurrentReleaseTag.outputs.exists }} == true ]]; then exit 1; fi - if: github.base_ref == 'unstable' || github.base_ref == 'stable' + if: >- + github.event.name == 'schedule' || github.base_ref == 'unstable' || + github.base_ref == 'stable' - name: Install Prettier run: npm install --save-dev --save-exact prettier - name: Pretty print changelog. @@ -472,8 +513,18 @@ jobs: tag_name: v${{fromJson(steps.changelog.outputs.content).version}} body: ${{fromJson(steps.changelog.outputs.content).body}} prerelease: ${{fromJson(steps.changelog.outputs.content).prerelease}} - draft: true - if: github.ref == 'refs/heads/unstable' || github.ref == 'refs/heads/stable' + draft: ${{ ! (github.event.name == 'schedule') }} + - uses: dev-drprasad/delete-older-releases@v0.2.0 + name: Remove Old Releases + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + keep_latest: ${{ env.NIGHTLIES_TO_KEEP }} + delete_tag_pattern: nightly + delete_tags: true + if: >- + github.event.name == 'schedule' || github.ref == 'refs/heads/unstable' || + github.ref == 'refs/heads/stable' needs: - version_assertions - lint @@ -495,6 +546,17 @@ jobs: uses: actions/download-artifact@v2 with: path: artifacts + - name: Setup nightly + env: + GITHUB_TOKEN: ${{ github.token }} + run: |2- + + node ./run nightly-gen --skip-version-validation + head CHANGELOG.md + cat config.json + + if: github.event.name == 'schedule' + shell: bash - name: Read changelog info id: changelog run: |2- @@ -539,7 +601,9 @@ jobs: aws s3 cp ./artifacts/content/assets/wasm_imports.js.gz s3://ensocdn/ide/${{fromJson(steps.changelog.outputs.content).version}}/wasm_imports.js.gz --profile s3-upload --acl public-read --content-encoding gzip - if: github.ref == 'refs/heads/unstable' || github.ref == 'refs/heads/stable' + if: >- + github.event.name == 'schedule' || github.ref == 'refs/heads/unstable' || + github.ref == 'refs/heads/stable' needs: - version_assertions - lint diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b5396a71..ff97bd71d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Next Release +
![New Features](/docs/assets/tags/new_features.svg) + +#### Visual Environment + +- [Nightly releases.][1834] After every workday, CI performs an IDE build and + publishes a nightly pre-release on GitHub. +
![Bug Fixes](/docs/assets/tags/bug_fixes.svg) #### Visual Environment @@ -13,6 +20,7 @@
[1776]: https://github.com/enso-org/ide/pull/1776 +[1834]: https://github.com/enso-org/ide/pull/1834 # Enso 2.0.0-alpha.17 (2021-09-23) diff --git a/build/github.js b/build/github.js new file mode 100644 index 000000000..8b5a774d0 --- /dev/null +++ b/build/github.js @@ -0,0 +1,31 @@ +const { Octokit } = require("@octokit/core") + +const organization = 'enso-org' +const engineRepo = 'enso' +const token = process.env.GITHUB_TOKEN +const octokit = new Octokit({ auth: token }) + +function isNightly(release) { + const nightlyInfix = "Nightly" + return release.name.indexOf(nightlyInfix) >= 0 && !release.draft +} + +async function fetchAllReleases(repo) { + const res = await octokit.request("GET /repos/{owner}/{repo}/releases", { + owner: organization, + repo: repo, + }) + return res.data +} + +async function fetchNightlies(repo) { + const releases = await fetchAllReleases(repo) + const nightlies = releases.filter(isNightly) + return nightlies +} + +async function fetchEngineNightlies() { + return await fetchNightlies(engineRepo) +} + +module.exports = { fetchEngineNightlies } diff --git a/build/package.json b/build/package.json index b1a62c5a5..53eb2fff0 100644 --- a/build/package.json +++ b/build/package.json @@ -8,6 +8,7 @@ "glob": "^7.1.6", "js-yaml": "4.0.0", "ncp": "^2.0.0", + "@octokit/core": "^3.5.0", "semver": "7.3.4", "unzipper": "^0.10.11", "yargs": "^15.3.0" diff --git a/build/paths.js b/build/paths.js index 7d1fba9ba..290b168a1 100644 --- a/build/paths.js +++ b/build/paths.js @@ -11,6 +11,9 @@ let paths = {} paths.root = path.dirname(__dirname) +paths.changelog = path.join(paths.root,'CHANGELOG.md') +paths.configJson = path.join(paths.root,'config.json') + paths.script = {} paths.script.main = path.join(paths.root,'run') paths.script.root = path.join(paths.root,'build') diff --git a/build/release.js b/build/release.js index 30e48665a..1d4c7f4d9 100644 --- a/build/release.js +++ b/build/release.js @@ -40,20 +40,21 @@ class Version { this.minor = minor this.patch = patch this.tag = tag - this.tagVersion = parseInt(tagVersion) + this.tagVersion = tagVersion this.rcTag = rcTag this.rcTagVersion = rcTagVersion } lt(that) { - if (this.major < that.major) { return true } - if (this.minor < that.minor) { return true } - if (this.patch < that.patch) { return true } - if (this.tag === 'alpha' && that.tag === 'beta') { return true } - if (this.tag === 'alpha' && that.tag === 'rc') { return true } - if (this.tag === 'beta' && that.tag === 'rc') { return true } - if (this.tagVersion < that.tagVersion) { return true } - if (this.rcTagVersion < that.rcTagVersion) { return true } + if (this.major < that.major) { return true } + if (this.minor < that.minor) { return true } + if (this.patch < that.patch) { return true } + if (this.tag === 'nightly' && that.tag !== 'nightly') { return false } + if (this.tag === 'alpha' && that.tag === 'beta') { return true } + if (this.tag === 'alpha' && that.tag === 'rc') { return true } + if (this.tag === 'beta' && that.tag === 'rc') { return true } + if (ltStrings(this.tagVersion, that.tagVersion)) { return true } + if (ltStrings(this.rcTagVersion, that.rcTagVersion)) { return true } return false } @@ -73,7 +74,14 @@ class Version { } } +/// Compare two versions lexicographically. +function ltStrings(version1, version2) { + let maxLength = Math.max(version1.length, version2.length) + let v1 = version1.padStart(maxLength, ' ') + let v2 = version2.padStart(maxLength, ' ') + return v1 < v2 +} // ====================== // === ChangelogEntry === @@ -148,7 +156,7 @@ function changelogEntries() { let version = new NextReleaseVersion entries.push(new ChangelogEntry(version,body)) } else { - let headerReg = /^ Enso (?[0-9]+)\.(?[0-9]+)\.(?[0-9]+)(-(?alpha|beta|rc)\.(?[0-9]+))?(.(?rc)\.(?[0-9]+))? \((?[0-9][0-9][0-9][0-9])-(?[0-9][0-9])-(?[0-9][0-9])\)/ + let headerReg = /^ Enso (?[0-9]+)\.(?[0-9]+)\.(?[0-9]+)(-(?alpha|beta|nightly|rc)\.(?[0-9-]+))?(.(?rc)\.(?[0-9]+))? \((?[0-9][0-9][0-9][0-9])-(?[0-9][0-9])-(?[0-9][0-9])\)/ let match = header.match(headerReg) if (!match) { throw `Improper changelog entry header: '${header}'. See the 'CHANGELOG_TEMPLATE.md' for details.` @@ -186,10 +194,25 @@ function currentVersion() { return changelog().currentVersion() } +/// Create the nightly version based on the last version in changelog. +function nightlyVersion() { + let changelog = new Changelog + let version = changelog.entries[0].version + if (version instanceof NextReleaseVersion) { + version = changelog.entries[1].version + } + return `${version.major}.${version.minor}.${version.patch}-nightly.${isoDate()}` +} + +/// Get the current ISO date in format `YYYY-MM-DD`. +function isoDate() { + let date = new Date() + return date.toISOString().split('T')[0] +} // =============== // === Exports === // =============== -module.exports = {ENGINE_VERSION,Version,NextReleaseVersion,changelog,currentVersion} +module.exports = {ENGINE_VERSION,Version,NextReleaseVersion,changelog,currentVersion,nightlyVersion,isoDate} diff --git a/build/run.js b/build/run.js index 2270a6e10..a69cd3639 100755 --- a/build/run.js +++ b/build/run.js @@ -4,6 +4,7 @@ const fs = require('fs').promises const fse = require('fs-extra') const fss = require('fs') const unzipper = require('unzipper') +const github = require('./github') const glob = require('glob') const ncp = require('ncp').ncp const os = require('os') @@ -284,7 +285,7 @@ commands.watch.common = async function(argv) { await cmd.with_cwd(paths.js.root, async () => { // Among other things, this will call the build script of the project-manager package. But // this is unnecessary because that script is already called by `build_project_manager` - // above. + // above. return commands.build.js(argv) }) @@ -328,6 +329,29 @@ commands.dist.js = async function() { } +// === Nightly Gen === +commands['nightly-gen'] = command(`Generate Nightly CI build related files`) +commands['nightly-gen'].rust = async function(argv) { + let nightlies = await github.fetchEngineNightlies() + let engineVersion = nightlies[0].name + let nightlyPrefix = 'Enso Nightly ' + if (engineVersion.startsWith(nightlyPrefix)) { + engineVersion = engineVersion.substring(nightlyPrefix.length) + } + + let config = require('../config.json') + config.engineVersion = engineVersion + + let nightlyVersion = release.nightlyVersion() + console.log(`engine version: ${engineVersion}`) + console.log(`IDE nightly version: ${nightlyVersion}`) + + // Update config.json + fss.writeFileSync(paths.configJson, JSON.stringify(config)) + // Update changelog + await cmd.run('sed', ["-i'.bak'", `'1s/.*/# Enso ${nightlyVersion} (${release.isoDate()})/'`, paths.changelog]) +} + // === CI Gen === /// The command is used by CI to generate the file `CURRENT_RELEASE_CHANGELOG.json`, which contains @@ -339,7 +363,7 @@ commands['ci-gen'].rust = async function(argv) { let body = entry.body let version = entry.version.toString() let prerelease = entry.isPrerelease() - let obj = {version,body,prerelease}; + let obj = {version,body,prerelease} let json = JSON.stringify(obj) fss.writeFileSync(path.join(paths.root,'CURRENT_RELEASE_CHANGELOG.json'),json) } diff --git a/build/workflow.js b/build/workflow.js index 8355558b5..73226ef66 100644 --- a/build/workflow.js +++ b/build/workflow.js @@ -25,6 +25,17 @@ const WINDOWS_RUNNER_GITHUB_HOSTED = ["windows-latest"] +// ================== +// === Conditions === +// ================== + +let nightlyReleaseCondition = `github.event.name == 'schedule'` +/// Make a release only if it was a push to 'unstable' or 'stable'. Even if it was a pull request +/// FROM these branches, the `github.ref` will be different. +let releaseCondition = `${nightlyReleaseCondition} || github.ref == 'refs/heads/unstable' || github.ref == 'refs/heads/stable'` + + + // ============= // === Utils === // ============= @@ -313,6 +324,20 @@ let getListOfChangedFiles = { // === Changelog === // ================= +let setupNightly = { + name: 'Setup nightly', + env: { + GITHUB_TOKEN: '${{ github.token }}', + }, + run: ` + node ./run nightly-gen --skip-version-validation + head CHANGELOG.md + cat config.json + `, + if: nightlyReleaseCondition, + shell: 'bash', +} + let getCurrentReleaseChangelogInfo = { name: 'Read changelog info', id: 'changelog', @@ -356,12 +381,23 @@ let uploadGitHubRelease = [ tag_name: "v${{fromJson(steps.changelog.outputs.content).version}}", body: "${{fromJson(steps.changelog.outputs.content).body}}", prerelease: "${{fromJson(steps.changelog.outputs.content).prerelease}}", - draft: true, + draft: `\${{ ! (${nightlyReleaseCondition}) }}`, }, } ] - +let deleteOlderNightlies = { + uses: 'dev-drprasad/delete-older-releases@v0.2.0', + name: 'Remove Old Releases', + env: { + GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}', + }, + with: { + keep_latest: '${{ env.NIGHTLIES_TO_KEEP }}', + delete_tag_pattern: 'nightly', + delete_tags: true, + }, +} // =================== // === CDN Release === @@ -432,7 +468,7 @@ let assertReleaseDoNotExists = [ { name: 'Fail if release already exists', run: 'if [[ ${{ steps.checkCurrentReleaseTag.outputs.exists }} == true ]]; then exit 1; fi', - if: `github.base_ref == 'unstable' || github.base_ref == 'stable'` + if: `${nightlyReleaseCondition} || github.base_ref == 'unstable' || github.base_ref == 'stable'` } ] @@ -455,24 +491,29 @@ let assertions = list( // === Workflow === // =============== -/// Make a release only if it was a push to 'unstable' or 'stable'. Even if it was a pull request -/// FROM these branches, the `github.ref` will be different. -let releaseCondition = `github.ref == 'refs/heads/unstable' || github.ref == 'refs/heads/stable'` - let workflow = { name : "GUI CI", + env: { + NIGHTLIES_TO_KEEP: 20, + }, on: { push: { - branches: ['develop','unstable','stable'] + branches: ['develop','unstable','stable'], }, pull_request: {}, - workflow_dispatch: {} + workflow_dispatch: {}, + schedule: [ + { + cron: '0 6 * * 2-6' + }, + ] }, jobs: { info: job_on_macos("Build Info", [ dumpGitHubContext ]), version_assertions: job_on_macos("Assertions", [ + setupNightly, getCurrentReleaseChangelogInfo, assertions ]), @@ -512,6 +553,7 @@ let workflow = { ( [MACOS_RUNNER_GITHUB_HOSTED,WINDOWS_RUNNER_GITHUB_HOSTED,cached_linux_runner("package")] , "Build package" , [ + setupNightly, getCurrentReleaseChangelogInfo, installNode, installTypeScript, @@ -528,17 +570,20 @@ let workflow = { ], { needs: ['build_wasm'] }), release_to_github: job_on_macos("GitHub Release", [ downloadArtifacts, + setupNightly, getCurrentReleaseChangelogInfo, // This assertion is checked earlier, but we should double-check it in case several // CI jobs wil be run on this branch and a release was created when this workflow was // running. assertReleaseDoNotExists, uploadGitHubRelease, + deleteOlderNightlies, ],{ if:releaseCondition, needs:['version_assertions','lint','test','package'] }), release_to_cdn: job_on_ubuntu_18_04("CDN Release", [ downloadArtifacts, + setupNightly, getCurrentReleaseChangelogInfo, prepareAwsSessionCDN, uploadToCDN('index.js.gz','style.css','ide.wasm','wasm_imports.js.gz'),