From a4a0272d32343534b36ae0393aa813309aa97d9d Mon Sep 17 00:00:00 2001 From: YR Chen Date: Wed, 1 Mar 2023 03:54:29 +0800 Subject: [PATCH 01/24] winget-source: init --- winget-source/Dockerfile | 7 + winget-source/package-lock.json | 1287 +++++++++++++++++++++++++++++++ winget-source/package.json | 15 + winget-source/sync-repo.js | 30 + winget-source/sync.sh | 16 + winget-source/utilities.js | 150 ++++ 6 files changed, 1505 insertions(+) create mode 100644 winget-source/Dockerfile create mode 100644 winget-source/package-lock.json create mode 100644 winget-source/package.json create mode 100644 winget-source/sync-repo.js create mode 100644 winget-source/sync.sh create mode 100644 winget-source/utilities.js diff --git a/winget-source/Dockerfile b/winget-source/Dockerfile new file mode 100644 index 0000000..f83a89d --- /dev/null +++ b/winget-source/Dockerfile @@ -0,0 +1,7 @@ +FROM ustcmirror/base:alpine +LABEL maintainer "YR Chen " + +ADD ["sync.sh", "sync-repo.js", "utilities.js", "package.json", "package-lock.json", "/"] + +RUN apk add --no-cache nodejs npm python3 +RUN cd / && npm install diff --git a/winget-source/package-lock.json b/winget-source/package-lock.json new file mode 100644 index 0000000..274c3cd --- /dev/null +++ b/winget-source/package-lock.json @@ -0,0 +1,1287 @@ +{ + "name": "winget-source", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "winget-source", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "async": "^3.2.4", + "jszip": "^3.10.1", + "node-fetch": "^3.3.0", + "sqlite3": "^5.1.4" + } + }, + "node_modules/@gar/promisify": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz", + "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==", + "optional": true + }, + "node_modules/@mapbox/node-pre-gyp": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.10.tgz", + "integrity": "sha512-4ySo4CjzStuprMwk35H5pPbkymjv1SF3jGLj6rAHp/xT/RF7TL7bd9CTm1xDY49K2qF7jmR/g7k+SkLETP6opA==", + "dependencies": { + "detect-libc": "^2.0.0", + "https-proxy-agent": "^5.0.0", + "make-dir": "^3.1.0", + "node-fetch": "^2.6.7", + "nopt": "^5.0.0", + "npmlog": "^5.0.1", + "rimraf": "^3.0.2", + "semver": "^7.3.5", + "tar": "^6.1.11" + }, + "bin": { + "node-pre-gyp": "bin/node-pre-gyp" + } + }, + "node_modules/@mapbox/node-pre-gyp/node_modules/node-fetch": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz", + "integrity": "sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/@npmcli/fs": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-1.1.1.tgz", + "integrity": "sha512-8KG5RD0GVP4ydEzRn/I4BNDuxDtqVbOdm8675T49OIG/NGhaK0pjPX7ZcDlvKYbA+ulvVK3ztfcF4uBdOxuJbQ==", + "optional": true, + "dependencies": { + "@gar/promisify": "^1.0.1", + "semver": "^7.3.5" + } + }, + "node_modules/@npmcli/move-file": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-1.1.2.tgz", + "integrity": "sha512-1SUf/Cg2GzGDyaf15aR9St9TWlb+XvbZXWpDx8YKs7MLzMH/BCeopv+y9vzrzgkfykCGuWOlSu3mZhj2+FQcrg==", + "deprecated": "This functionality has been moved to @npmcli/fs", + "optional": true, + "dependencies": { + "mkdirp": "^1.0.4", + "rimraf": "^3.0.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@tootallnate/once": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", + "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==", + "optional": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" + }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/agentkeepalive": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.2.1.tgz", + "integrity": "sha512-Zn4cw2NEqd+9fiSVWMscnjyQ1a8Yfoc5oBajLeo5w+YBHgDUcEBY2hS4YpTz6iN5f/2zQiktcuM6tS8x1p9dpA==", + "optional": true, + "dependencies": { + "debug": "^4.1.0", + "depd": "^1.1.2", + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/aggregate-error": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", + "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", + "optional": true, + "dependencies": { + "clean-stack": "^2.0.0", + "indent-string": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/aproba": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", + "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==" + }, + "node_modules/are-we-there-yet": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz", + "integrity": "sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==", + "dependencies": { + "delegates": "^1.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/async": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", + "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/cacache": { + "version": "15.3.0", + "resolved": "https://registry.npmjs.org/cacache/-/cacache-15.3.0.tgz", + "integrity": "sha512-VVdYzXEn+cnbXpFgWs5hTT7OScegHVmLhJIR8Ufqk3iFD6A6j5iSX1KuBTfNEv4tdJWE2PzA6IVFtcLC7fN9wQ==", + "optional": true, + "dependencies": { + "@npmcli/fs": "^1.0.0", + "@npmcli/move-file": "^1.0.1", + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "glob": "^7.1.4", + "infer-owner": "^1.0.4", + "lru-cache": "^6.0.0", + "minipass": "^3.1.1", + "minipass-collect": "^1.0.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.2", + "mkdirp": "^1.0.3", + "p-map": "^4.0.0", + "promise-inflight": "^1.0.1", + "rimraf": "^3.0.2", + "ssri": "^8.0.1", + "tar": "^6.0.2", + "unique-filename": "^1.1.1" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "engines": { + "node": ">=10" + } + }, + "node_modules/clean-stack": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", + "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", + "optional": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/color-support": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", + "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", + "bin": { + "color-support": "bin.js" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + }, + "node_modules/console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==" + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "engines": { + "node": ">= 12" + } + }, + "node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==" + }, + "node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", + "optional": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/detect-libc": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", + "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/encoding": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", + "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "optional": true, + "dependencies": { + "iconv-lite": "^0.6.2" + } + }, + "node_modules/env-paths": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "optional": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/err-code": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz", + "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==", + "optional": true + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" + }, + "node_modules/gauge": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-3.0.2.tgz", + "integrity": "sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==", + "dependencies": { + "aproba": "^1.0.3 || ^2.0.0", + "color-support": "^1.1.2", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.1", + "object-assign": "^4.1.1", + "signal-exit": "^3.0.0", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1", + "wide-align": "^1.1.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", + "optional": true + }, + "node_modules/has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==" + }, + "node_modules/http-cache-semantics": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz", + "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==", + "optional": true + }, + "node_modules/http-proxy-agent": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", + "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "optional": true, + "dependencies": { + "@tootallnate/once": "1", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "optional": true, + "dependencies": { + "ms": "^2.0.0" + } + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "optional": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==" + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "optional": true, + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/infer-owner": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/infer-owner/-/infer-owner-1.0.4.tgz", + "integrity": "sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==", + "optional": true + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/ip": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz", + "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==", + "optional": true + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-lambda": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-lambda/-/is-lambda-1.0.1.tgz", + "integrity": "sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==", + "optional": true + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "optional": true + }, + "node_modules/jszip": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", + "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, + "node_modules/jszip/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/jszip/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/jszip/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "dependencies": { + "immediate": "~3.0.5" + } + }, + "node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dependencies": { + "semver": "^6.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/make-fetch-happen": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-9.1.0.tgz", + "integrity": "sha512-+zopwDy7DNknmwPQplem5lAZX/eCOzSvSNNcSKm5eVwTkOBzoktEfXsa9L23J/GIRhxRsaxzkPEhrJEpE2F4Gg==", + "optional": true, + "dependencies": { + "agentkeepalive": "^4.1.3", + "cacache": "^15.2.0", + "http-cache-semantics": "^4.1.0", + "http-proxy-agent": "^4.0.1", + "https-proxy-agent": "^5.0.0", + "is-lambda": "^1.0.1", + "lru-cache": "^6.0.0", + "minipass": "^3.1.3", + "minipass-collect": "^1.0.2", + "minipass-fetch": "^1.3.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "negotiator": "^0.6.2", + "promise-retry": "^2.0.1", + "socks-proxy-agent": "^6.0.0", + "ssri": "^8.0.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-collect": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-1.0.2.tgz", + "integrity": "sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minipass-fetch": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-1.4.1.tgz", + "integrity": "sha512-CGH1eblLq26Y15+Azk7ey4xh0J/XfJfrCox5LDJiKqI2Q2iwOLOKrlmIaODiSQS8d18jalF6y2K2ePUm0CmShw==", + "optional": true, + "dependencies": { + "minipass": "^3.1.0", + "minipass-sized": "^1.0.3", + "minizlib": "^2.0.0" + }, + "engines": { + "node": ">=8" + }, + "optionalDependencies": { + "encoding": "^0.1.12" + } + }, + "node_modules/minipass-flush": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.5.tgz", + "integrity": "sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minipass-pipeline": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz", + "integrity": "sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-sized": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/minipass-sized/-/minipass-sized-1.0.3.tgz", + "integrity": "sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g==", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dependencies": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "optional": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-addon-api": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-4.3.0.tgz", + "integrity": "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ==" + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.0.tgz", + "integrity": "sha512-BKwRP/O0UvoMKp7GNdwPlObhYGB5DQqwhEDQlNKuoqwVYSxkSZCSbHjnFFmUEtwSKRPU4kNK8PbDYYitwaE3QA==", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/node-gyp": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-8.4.1.tgz", + "integrity": "sha512-olTJRgUtAb/hOXG0E93wZDs5YiJlgbXxTwQAFHyNlRsXQnYzUaF2aGgujZbw+hR8aF4ZG/rST57bWMWD16jr9w==", + "optional": true, + "dependencies": { + "env-paths": "^2.2.0", + "glob": "^7.1.4", + "graceful-fs": "^4.2.6", + "make-fetch-happen": "^9.1.0", + "nopt": "^5.0.0", + "npmlog": "^6.0.0", + "rimraf": "^3.0.2", + "semver": "^7.3.5", + "tar": "^6.1.2", + "which": "^2.0.2" + }, + "bin": { + "node-gyp": "bin/node-gyp.js" + }, + "engines": { + "node": ">= 10.12.0" + } + }, + "node_modules/node-gyp/node_modules/are-we-there-yet": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-3.0.1.tgz", + "integrity": "sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==", + "optional": true, + "dependencies": { + "delegates": "^1.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/node-gyp/node_modules/gauge": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-4.0.4.tgz", + "integrity": "sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg==", + "optional": true, + "dependencies": { + "aproba": "^1.0.3 || ^2.0.0", + "color-support": "^1.1.3", + "console-control-strings": "^1.1.0", + "has-unicode": "^2.0.1", + "signal-exit": "^3.0.7", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1", + "wide-align": "^1.1.5" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/node-gyp/node_modules/npmlog": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-6.0.2.tgz", + "integrity": "sha512-/vBvz5Jfr9dT/aFWd0FIRf+T/Q2WBsLENygUaFUqstqsycmZAP/t5BvFJTK0viFmSUxiUKTUplWy5vt+rvKIxg==", + "optional": true, + "dependencies": { + "are-we-there-yet": "^3.0.0", + "console-control-strings": "^1.1.0", + "gauge": "^4.0.3", + "set-blocking": "^2.0.0" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/nopt": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", + "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", + "dependencies": { + "abbrev": "1" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/npmlog": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-5.0.1.tgz", + "integrity": "sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==", + "dependencies": { + "are-we-there-yet": "^2.0.0", + "console-control-strings": "^1.1.0", + "gauge": "^3.0.0", + "set-blocking": "^2.0.0" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "optional": true, + "dependencies": { + "aggregate-error": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "node_modules/promise-inflight": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz", + "integrity": "sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==", + "optional": true + }, + "node_modules/promise-retry": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz", + "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", + "optional": true, + "dependencies": { + "err-code": "^2.0.2", + "retry": "^0.12.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/readable-stream": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.1.tgz", + "integrity": "sha512-+rQmrWMYGA90yenhTYsLWAsLsqVC8osOw6PKE1HDYiO0gdPeKe/xDHNzIAIn4C91YQ6oenEhfYqqc1883qHbjQ==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "optional": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "optional": true + }, + "node_modules/semver": { + "version": "7.3.8", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", + "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==" + }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==" + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" + }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "optional": true, + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.7.1.tgz", + "integrity": "sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==", + "optional": true, + "dependencies": { + "ip": "^2.0.0", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.13.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-6.2.1.tgz", + "integrity": "sha512-a6KW9G+6B3nWZ1yB8G7pJwL3ggLy1uTzKAgCb7ttblwqdz9fMGJUuTy3uFzEP48FAs9FLILlmzDlE2JJhVQaXQ==", + "optional": true, + "dependencies": { + "agent-base": "^6.0.2", + "debug": "^4.3.3", + "socks": "^2.6.2" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/sqlite3": { + "version": "5.1.4", + "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-5.1.4.tgz", + "integrity": "sha512-i0UlWAzPlzX3B5XP2cYuhWQJsTtlMD6obOa1PgeEQ4DHEXUuyJkgv50I3isqZAP5oFc2T8OFvakmDh2W6I+YpA==", + "hasInstallScript": true, + "dependencies": { + "@mapbox/node-pre-gyp": "^1.0.0", + "node-addon-api": "^4.2.0", + "tar": "^6.1.11" + }, + "optionalDependencies": { + "node-gyp": "8.x" + }, + "peerDependencies": { + "node-gyp": "8.x" + }, + "peerDependenciesMeta": { + "node-gyp": { + "optional": true + } + } + }, + "node_modules/ssri": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/ssri/-/ssri-8.0.1.tgz", + "integrity": "sha512-97qShzy1AiyxvPNIkLWoGua7xoQzzPjQ0HAH4B0rWKo7SZ6USuPcrUiAFrws0UH8RrbWmgq3LMTObhPIHbbBeQ==", + "optional": true, + "dependencies": { + "minipass": "^3.1.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tar": { + "version": "6.1.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.13.tgz", + "integrity": "sha512-jdIBIN6LTIe2jqzay/2vtYLlBHa3JF42ot3h1dW8Q0PaAG4v8rm0cvpVePtau5C6OKXGGcgO9q2AMNSWxiLqKw==", + "dependencies": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^4.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/tar/node_modules/minipass": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-4.2.4.tgz", + "integrity": "sha512-lwycX3cBMTvcejsHITUgYj6Gy6A7Nh4Q6h9NP4sTHY1ccJlC7yKzDmiShEHsJ16Jf1nKGDEaiHxiltsJEvk0nQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, + "node_modules/unique-filename": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz", + "integrity": "sha512-Vmp0jIp2ln35UTXuryvjzkjGdRyf9b2lTXuSYUiPmzRcl3FDtYqAwOnTJkAngD9SWhnoJzDbTKwaOrZ+STtxNQ==", + "optional": true, + "dependencies": { + "unique-slug": "^2.0.0" + } + }, + "node_modules/unique-slug": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-2.0.2.tgz", + "integrity": "sha512-zoWr9ObaxALD3DOPfjPSqxt4fnZiWblxHIgeWqW8x7UqDzEtHEQLzji2cuJYQFCU6KmoJikOYAZlrTHHebjx2w==", + "optional": true, + "dependencies": { + "imurmurhash": "^0.1.4" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, + "node_modules/web-streams-polyfill": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", + "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", + "engines": { + "node": ">= 8" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "optional": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wide-align": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", + "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==", + "dependencies": { + "string-width": "^1.0.2 || 2 || 3 || 4" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + } + } +} diff --git a/winget-source/package.json b/winget-source/package.json new file mode 100644 index 0000000..44ac1c0 --- /dev/null +++ b/winget-source/package.json @@ -0,0 +1,15 @@ +{ + "name": "winget-source", + "version": "1.0.0", + "description": "Sync with WinGet source repository.", + "main": "sync-repo.js", + "author": "YR Chen ", + "license": "MIT", + "type": "module", + "dependencies": { + "async": "^3.2.4", + "jszip": "^3.10.1", + "node-fetch": "^3.3.0", + "sqlite3": "^5.1.4" + } +} diff --git a/winget-source/sync-repo.js b/winget-source/sync-repo.js new file mode 100644 index 0000000..e6c2ce6 --- /dev/null +++ b/winget-source/sync-repo.js @@ -0,0 +1,30 @@ +import async from 'async' +import sqlite3 from 'sqlite3' + +import { + buildPathpartMap, + buildUriList, + checkEnvironmentVariables, + exatractDBFromBundle, + getLocalPath, + makeTempDirectory, + syncFile +} from './utilities.js' + +const { parallelLimit } = checkEnvironmentVariables(); + +syncFile('source.msix').then(async _ => { + const temp = await makeTempDirectory('winget-repo-'); + const database = await exatractDBFromBundle(getLocalPath('source.msix'), temp); + const db = new sqlite3.Database(database, sqlite3.OPEN_READONLY); + + db.all('SELECT * FROM pathparts', (error, rows) => { + const pathparts = buildPathpartMap(error, rows); + db.all('SELECT pathpart FROM manifest', (error, rows) => { + db.close(); + const uris = buildUriList(error, rows, pathparts); + const tasks = uris.map(uri => async () => { await syncFile(uri); return; }); + async.parallelLimit(tasks, parallelLimit); + }); + }); +}); diff --git a/winget-source/sync.sh b/winget-source/sync.sh new file mode 100644 index 0000000..1895880 --- /dev/null +++ b/winget-source/sync.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +#WINGET_REPO_URL= +#WINGET_REPO_JOBS= + +## EXPORTED IN entry.sh +#TO= +#LOGDIR= +#LOGFILE= + +## SET IN ENVIRONMENT VARIABLES +#DEBUG= + +set -e + +exec node /sync-repo.js diff --git a/winget-source/utilities.js b/winget-source/utilities.js new file mode 100644 index 0000000..e4bd497 --- /dev/null +++ b/winget-source/utilities.js @@ -0,0 +1,150 @@ +import crypto from 'crypto' +import JSZip from 'jszip' +import fetch from 'node-fetch' +import os from 'os' +import path from 'path' +import process from 'process' + +import { existsSync } from 'fs' +import { mkdir, mkdtemp, readFile, utimes, writeFile } from 'fs/promises' + +const remote = process.env.WINGET_REPO_URL; +const local = process.env.TO; +const parallelLimit = parseInt(process.env.WINGET_REPO_JOBS ?? 8); + +/** + * @param {number} id + * @param {Map} pathparts + * + * @returns {string} + */ +function resolvePathpart(id, pathparts) { + const pathpart = pathparts.get(id); + if (pathpart === undefined) return ''; + return path.posix.join(resolvePathpart(pathpart.parent, pathparts), pathpart.pathpart); +} + +/** + * @param {crypto.BinaryLike} buffer + * @param {crypto.BinaryToTextEncoding} encoding + * + * @returns {string} + */ +function computeMD5(buffer, encoding) { + const hash = crypto.createHash('md5'); + hash.update(buffer, 'utf-8'); + return hash.digest(encoding); +} + +/** + * @param {Error?} error + * @param {{ rowid: number, parent: number, pathpart: string }[]} rows + * + * @returns {Map} + */ +export function buildPathpartMap(error, rows) { + if (error) { + console.error(error); + process.exit(-1); + } + return new Map(rows.map(row => + [row.rowid, { parent: row.parent, pathpart: row.pathpart }] + )); +} + +/** + * @param {Error?} error + * @param {{ pathpart: string, [key: string]: string }[]} rows + * @param {Map} pathparts + * + * @returns {string[]} + */ +export function buildUriList(error, rows, pathparts) { + if (error) { + console.error(error); + process.exit(-1); + } + return rows.map(row => resolvePathpart(row.pathpart, pathparts)); +} + +export function checkEnvironmentVariables() { + if (!remote || !local || !parallelLimit) { + console.error("required envirenent variable(s) not set!"); + process.exit(-1); + } + return { remote, local, parallelLimit }; +} + +/** + * @param {fs.PathLike} msixPath + * @param {fs.PathLike} tempDirectory + * + * @returns {Promise} + */ +export async function exatractDBFromBundle(msixPath, tempDirectory) { + const bundle = await readFile(msixPath); + const zip = await JSZip.loadAsync(bundle); + const buffer = await zip.file(path.posix.join('Public', 'index.db')).async('Uint8Array'); + const destination = path.join(tempDirectory, 'index.db'); + await writeFile(destination, buffer); + return destination; +} + +/** + * @param {string} uri + * + * @returns {string} + */ +export function getLocalPath(uri) { + return path.join(local, uri); +} + +/** + * @param {string} uri + * + * @returns {URL} + */ +export function getRemoteURL(uri) { + return new URL(`${remote}/${uri}`); +} + +/** + * @param {string} prefix + * + * @returns {Promise} + */ +export async function makeTempDirectory(prefix) { + return await mkdtemp(path.join(os.tmpdir(), prefix)); +} + +/** + * @param {string} uri + * + * @returns {Promise} + */ +export async function syncFile(uri) { + const localPath = getLocalPath(uri); + const remoteURL = getRemoteURL(uri); + await mkdir(path.dirname(localPath), { recursive: true }); + if (existsSync(localPath)) { + const response = await fetch(remoteURL, { method: 'HEAD' }); + const latestMD5 = response.headers.get('Content-MD5'); + if (latestMD5) { + const buffer = await readFile(localPath); + const md5 = computeMD5(buffer, 'base64'); + if (latestMD5 === md5) { + console.info(`skipped ${uri} because it's up to date`); + return false; + } + } + } + const response = await fetch(remoteURL); + console.log(`downloaded ${uri}`); + await writeFile(localPath, response.body); + const lastModified = response.headers.get('Last-Modified'); + if (lastModified) { + const date = new Date(lastModified); + await utimes(localPath, date, date); + } + return true; +} From 5f1eec77c1731983b38c3eeeb11aa5be27bda347 Mon Sep 17 00:00:00 2001 From: YR Chen Date: Wed, 1 Mar 2023 05:13:03 +0800 Subject: [PATCH 02/24] winget-source: cleans up temp directory before exit --- winget-source/sync-repo.js | 10 ++++++++-- winget-source/utilities.js | 1 - 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/winget-source/sync-repo.js b/winget-source/sync-repo.js index e6c2ce6..0d0c39e 100644 --- a/winget-source/sync-repo.js +++ b/winget-source/sync-repo.js @@ -1,5 +1,6 @@ import async from 'async' import sqlite3 from 'sqlite3' +import { rm } from 'fs/promises' import { buildPathpartMap, @@ -23,8 +24,13 @@ syncFile('source.msix').then(async _ => { db.all('SELECT pathpart FROM manifest', (error, rows) => { db.close(); const uris = buildUriList(error, rows, pathparts); - const tasks = uris.map(uri => async () => { await syncFile(uri); return; }); - async.parallelLimit(tasks, parallelLimit); + async.eachLimit(uris, parallelLimit, syncFile, (error) => { + rm(temp, { recursive: true }); + if (error) { + console.error(error); + process.exit(-1); + } + }); }); }); }); diff --git a/winget-source/utilities.js b/winget-source/utilities.js index e4bd497..b293307 100644 --- a/winget-source/utilities.js +++ b/winget-source/utilities.js @@ -4,7 +4,6 @@ import fetch from 'node-fetch' import os from 'os' import path from 'path' import process from 'process' - import { existsSync } from 'fs' import { mkdir, mkdtemp, readFile, utimes, writeFile } from 'fs/promises' From a7f120413caf81b5c18ad86a160784b7857702e0 Mon Sep 17 00:00:00 2001 From: YR Chen Date: Wed, 1 Mar 2023 05:18:36 +0800 Subject: [PATCH 03/24] winget-source: update package.json --- winget-source/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/winget-source/package.json b/winget-source/package.json index 44ac1c0..ef0f8b7 100644 --- a/winget-source/package.json +++ b/winget-source/package.json @@ -1,7 +1,7 @@ { "name": "winget-source", "version": "1.0.0", - "description": "Sync with WinGet source repository.", + "description": "Sync with pre-indexed WinGet source repository.", "main": "sync-repo.js", "author": "YR Chen ", "license": "MIT", From ff9fc018988dd867511d9419c11fce0a1b16dee4 Mon Sep 17 00:00:00 2001 From: YR Chen Date: Wed, 1 Mar 2023 13:04:12 +0800 Subject: [PATCH 04/24] winget-source: improves Dockerized performance --- winget-source/Dockerfile | 9 +++++++-- winget-source/package.json | 3 +++ winget-source/sync.sh | 2 ++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/winget-source/Dockerfile b/winget-source/Dockerfile index f83a89d..2859b70 100644 --- a/winget-source/Dockerfile +++ b/winget-source/Dockerfile @@ -3,5 +3,10 @@ LABEL maintainer "YR Chen " ADD ["sync.sh", "sync-repo.js", "utilities.js", "package.json", "package-lock.json", "/"] -RUN apk add --no-cache nodejs npm python3 -RUN cd / && npm install +RUN apk add --no-cache nodejs + +RUN cd / \ + && apk add --no-cache npm python3 \ + && npm ci --only=production \ + && npm cache clean --force \ + && apk del --purge --no-cache npm python3 diff --git a/winget-source/package.json b/winget-source/package.json index ef0f8b7..4d5c34c 100644 --- a/winget-source/package.json +++ b/winget-source/package.json @@ -11,5 +11,8 @@ "jszip": "^3.10.1", "node-fetch": "^3.3.0", "sqlite3": "^5.1.4" + }, + "scripts": { + "start": "node sync-repo.js" } } diff --git a/winget-source/sync.sh b/winget-source/sync.sh index 1895880..401cfe2 100644 --- a/winget-source/sync.sh +++ b/winget-source/sync.sh @@ -11,6 +11,8 @@ ## SET IN ENVIRONMENT VARIABLES #DEBUG= +export NODE_ENV=production + set -e exec node /sync-repo.js From 15b9e9659f61a7f1e66504cabd895d6bc6d31489 Mon Sep 17 00:00:00 2001 From: YR Chen Date: Wed, 1 Mar 2023 13:16:43 +0800 Subject: [PATCH 05/24] winget-source: use modified time instead of MD5 --- winget-source/utilities.js | 58 +++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/winget-source/utilities.js b/winget-source/utilities.js index b293307..e4bd84b 100644 --- a/winget-source/utilities.js +++ b/winget-source/utilities.js @@ -1,16 +1,43 @@ -import crypto from 'crypto' import JSZip from 'jszip' import fetch from 'node-fetch' import os from 'os' import path from 'path' import process from 'process' import { existsSync } from 'fs' -import { mkdir, mkdtemp, readFile, utimes, writeFile } from 'fs/promises' +import { mkdir, mkdtemp, readFile, stat, utimes, writeFile } from 'fs/promises' const remote = process.env.WINGET_REPO_URL; const local = process.env.TO; const parallelLimit = parseInt(process.env.WINGET_REPO_JOBS ?? 8); +/** + * @param {Response} response + * + * @returns {Date | undefined} + */ +function getLastModifiedDate(response) { + const lastModified = response.headers.get('Last-Modified'); + if (lastModified) { + return new Date(Date.parse(lastModified)); + } else { + return undefined; + } +} + +/** + * @param {Response} response + * + * @returns {number} + */ +function getContentLength(response) { + const length = response.headers.get('Content-Length'); + if (length) { + return parseInt(length); + } else { + return 0; + } +} + /** * @param {number} id * @param {Map} pathparts @@ -23,18 +50,6 @@ function resolvePathpart(id, pathparts) { return path.posix.join(resolvePathpart(pathpart.parent, pathparts), pathpart.pathpart); } -/** - * @param {crypto.BinaryLike} buffer - * @param {crypto.BinaryToTextEncoding} encoding - * - * @returns {string} - */ -function computeMD5(buffer, encoding) { - const hash = crypto.createHash('md5'); - hash.update(buffer, 'utf-8'); - return hash.digest(encoding); -} - /** * @param {Error?} error * @param {{ rowid: number, parent: number, pathpart: string }[]} rows @@ -127,11 +142,11 @@ export async function syncFile(uri) { await mkdir(path.dirname(localPath), { recursive: true }); if (existsSync(localPath)) { const response = await fetch(remoteURL, { method: 'HEAD' }); - const latestMD5 = response.headers.get('Content-MD5'); - if (latestMD5) { - const buffer = await readFile(localPath); - const md5 = computeMD5(buffer, 'base64'); - if (latestMD5 === md5) { + const lastModified = getLastModifiedDate(response); + const contentLength = getContentLength(response); + if (lastModified) { + const localFile = await stat(localPath); + if (localFile.mtime.getTime() == lastModified.getTime() && localFile.size == contentLength) { console.info(`skipped ${uri} because it's up to date`); return false; } @@ -140,10 +155,9 @@ export async function syncFile(uri) { const response = await fetch(remoteURL); console.log(`downloaded ${uri}`); await writeFile(localPath, response.body); - const lastModified = response.headers.get('Last-Modified'); + const lastModified = getLastModifiedDate(response); if (lastModified) { - const date = new Date(lastModified); - await utimes(localPath, date, date); + await utimes(localPath, lastModified, lastModified); } return true; } From 7b79c69da74add3af3e88d69c22bb23d0ec8df3c Mon Sep 17 00:00:00 2001 From: YR Chen Date: Wed, 1 Mar 2023 14:35:11 +0800 Subject: [PATCH 06/24] winget-source: better naming --- winget-source/sync-repo.js | 8 ++++---- winget-source/utilities.js | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/winget-source/sync-repo.js b/winget-source/sync-repo.js index 0d0c39e..00e01f1 100644 --- a/winget-source/sync-repo.js +++ b/winget-source/sync-repo.js @@ -4,9 +4,9 @@ import { rm } from 'fs/promises' import { buildPathpartMap, - buildUriList, + buildURIList, checkEnvironmentVariables, - exatractDBFromBundle, + extractDatabaseFromBundle, getLocalPath, makeTempDirectory, syncFile @@ -16,14 +16,14 @@ const { parallelLimit } = checkEnvironmentVariables(); syncFile('source.msix').then(async _ => { const temp = await makeTempDirectory('winget-repo-'); - const database = await exatractDBFromBundle(getLocalPath('source.msix'), temp); + const database = await extractDatabaseFromBundle(getLocalPath('source.msix'), temp); const db = new sqlite3.Database(database, sqlite3.OPEN_READONLY); db.all('SELECT * FROM pathparts', (error, rows) => { const pathparts = buildPathpartMap(error, rows); db.all('SELECT pathpart FROM manifest', (error, rows) => { db.close(); - const uris = buildUriList(error, rows, pathparts); + const uris = buildURIList(error, rows, pathparts); async.eachLimit(uris, parallelLimit, syncFile, (error) => { rm(temp, { recursive: true }); if (error) { diff --git a/winget-source/utilities.js b/winget-source/utilities.js index e4bd84b..64b8dda 100644 --- a/winget-source/utilities.js +++ b/winget-source/utilities.js @@ -73,7 +73,7 @@ export function buildPathpartMap(error, rows) { * * @returns {string[]} */ -export function buildUriList(error, rows, pathparts) { +export function buildURIList(error, rows, pathparts) { if (error) { console.error(error); process.exit(-1); @@ -91,15 +91,15 @@ export function checkEnvironmentVariables() { /** * @param {fs.PathLike} msixPath - * @param {fs.PathLike} tempDirectory + * @param {fs.PathLike} directory * * @returns {Promise} */ -export async function exatractDBFromBundle(msixPath, tempDirectory) { +export async function extractDatabaseFromBundle(msixPath, directory) { const bundle = await readFile(msixPath); const zip = await JSZip.loadAsync(bundle); const buffer = await zip.file(path.posix.join('Public', 'index.db')).async('Uint8Array'); - const destination = path.join(tempDirectory, 'index.db'); + const destination = path.join(directory, 'index.db'); await writeFile(destination, buffer); return destination; } From 900f1217bb9239740305f66f072bc1046b9a12db Mon Sep 17 00:00:00 2001 From: YR Chen Date: Wed, 8 Mar 2023 00:36:50 +0800 Subject: [PATCH 07/24] winget-source: more robust `getRemoteURL()` --- winget-source/utilities.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/winget-source/utilities.js b/winget-source/utilities.js index 64b8dda..cb7dda1 100644 --- a/winget-source/utilities.js +++ b/winget-source/utilities.js @@ -119,7 +119,9 @@ export function getLocalPath(uri) { * @returns {URL} */ export function getRemoteURL(uri) { - return new URL(`${remote}/${uri}`); + const remoteURL = new URL(remote); + remoteURL.pathname = path.posix.join(remoteURL.pathname, uri); + return remoteURL; } /** From 2d0215ed0e2aed55d5a94bde36820ae45536a0f8 Mon Sep 17 00:00:00 2001 From: YR Chen Date: Wed, 8 Mar 2023 01:43:09 +0800 Subject: [PATCH 08/24] winget-source: JSDoc-styled documentation --- winget-source/sync-repo.js | 4 +- winget-source/utilities.js | 86 +++++++++++++++++++++++++------------- 2 files changed, 60 insertions(+), 30 deletions(-) diff --git a/winget-source/sync-repo.js b/winget-source/sync-repo.js index 00e01f1..f44ac7a 100644 --- a/winget-source/sync-repo.js +++ b/winget-source/sync-repo.js @@ -5,14 +5,14 @@ import { rm } from 'fs/promises' import { buildPathpartMap, buildURIList, - checkEnvironmentVariables, extractDatabaseFromBundle, getLocalPath, makeTempDirectory, + requireEnvironmentVariables, syncFile } from './utilities.js' -const { parallelLimit } = checkEnvironmentVariables(); +const { parallelLimit } = requireEnvironmentVariables(); syncFile('source.msix').then(async _ => { const temp = await makeTempDirectory('winget-repo-'); diff --git a/winget-source/utilities.js b/winget-source/utilities.js index cb7dda1..987cc58 100644 --- a/winget-source/utilities.js +++ b/winget-source/utilities.js @@ -6,14 +6,21 @@ import process from 'process' import { existsSync } from 'fs' import { mkdir, mkdtemp, readFile, stat, utimes, writeFile } from 'fs/promises' +/** The remote URL of a pre-indexed WinGet source repository. */ const remote = process.env.WINGET_REPO_URL; + +/** The local path to serve as the root of WinGet source repository. */ const local = process.env.TO; + +/** Maximum sync jobs to be executed in parallel. Defaults to 8. */ const parallelLimit = parseInt(process.env.WINGET_REPO_JOBS ?? 8); /** - * @param {Response} response + * Get last modified date from HTTP response headers. * - * @returns {Date | undefined} + * @param {Response} response The HTTP `fetch` response to parse. + * + * @returns {Date | undefined} Last modified date derived from the response, if exists. */ function getLastModifiedDate(response) { const lastModified = response.headers.get('Last-Modified'); @@ -25,9 +32,11 @@ function getLastModifiedDate(response) { } /** - * @param {Response} response + * Get content length from HTTP response headers. + * + * @param {Response} response The HTTP `fetch` response to parse. * - * @returns {number} + * @returns {number} Content length derived from the response, in bytes. */ function getContentLength(response) { const length = response.headers.get('Content-Length'); @@ -39,10 +48,12 @@ function getContentLength(response) { } /** - * @param {number} id - * @param {Map} pathparts + * Resolve path parts against the local storage. * - * @returns {string} + * @param {number} id The ID of the target path part. + * @param {Map} pathparts Path part storage built from the database. + * + * @returns {string} Full URI resolved from the given ID. */ function resolvePathpart(id, pathparts) { const pathpart = pathparts.get(id); @@ -51,10 +62,12 @@ function resolvePathpart(id, pathparts) { } /** - * @param {Error?} error - * @param {{ rowid: number, parent: number, pathpart: string }[]} rows + * Build a local storage for path parts from database query. + * + * @param {Error?} error Database error thrown from the query, if any. + * @param {{ rowid: number, parent: number, pathpart: string }[]} rows Rows returned by the query. * - * @returns {Map} + * @returns {Map} In-memory path part storage to query against. */ export function buildPathpartMap(error, rows) { if (error) { @@ -67,11 +80,13 @@ export function buildPathpartMap(error, rows) { } /** - * @param {Error?} error - * @param {{ pathpart: string, [key: string]: string }[]} rows - * @param {Map} pathparts + * Build a list of all manifest URIs from database query. * - * @returns {string[]} + * @param {Error?} error Database error thrown from the query, if any. + * @param {{ pathpart: string, [key: string]: string }[]} rows Rows returned by the query. + * @param {Map} pathparts Path part storage built from the database. + * + * @returns {string[]} Manifest URIs to sync. */ export function buildURIList(error, rows, pathparts) { if (error) { @@ -81,19 +96,26 @@ export function buildURIList(error, rows, pathparts) { return rows.map(row => resolvePathpart(row.pathpart, pathparts)); } -export function checkEnvironmentVariables() { +/** + * Check and return the required environment variables. + * + * @returns Environment variables to be used in the program. + */ +export function requireEnvironmentVariables() { if (!remote || !local || !parallelLimit) { - console.error("required envirenent variable(s) not set!"); + console.error("required environment variable(s) not set!"); process.exit(-1); } return { remote, local, parallelLimit }; } /** - * @param {fs.PathLike} msixPath - * @param {fs.PathLike} directory + * Extract database file from the source bundle. * - * @returns {Promise} + * @param {fs.PathLike} msixPath Path of the MSIX bundle file. + * @param {fs.PathLike} directory Path of directory to save the file. + * + * @returns {Promise} Path of the extracted `index.db` file. */ export async function extractDatabaseFromBundle(msixPath, directory) { const bundle = await readFile(msixPath); @@ -105,18 +127,22 @@ export async function extractDatabaseFromBundle(msixPath, directory) { } /** - * @param {string} uri + * Get the local sync path of a manifest. + * + * @param {string} uri Manifest URI. * - * @returns {string} + * @returns {string} Expected local path of the manifest file. */ export function getLocalPath(uri) { return path.join(local, uri); } /** - * @param {string} uri + * Get the remote URL of a manifest. * - * @returns {URL} + * @param {string} uri Manifest URI. + * + * @returns {URL} Remote URL to get the manifest from. */ export function getRemoteURL(uri) { const remoteURL = new URL(remote); @@ -125,18 +151,22 @@ export function getRemoteURL(uri) { } /** - * @param {string} prefix + * Create a unique temporary directory with given prefix. + * + * @param {string} prefix Temporary directory name prefix. Must not contain path separators. * - * @returns {Promise} + * @returns {Promise} Path to the created temporary directory. */ export async function makeTempDirectory(prefix) { return await mkdtemp(path.join(os.tmpdir(), prefix)); } /** - * @param {string} uri + * Sync a file with the remote server asynchronously. + * + * @param {string} uri URI to sync. * - * @returns {Promise} + * @returns {Promise} If the file is new or updated. */ export async function syncFile(uri) { const localPath = getLocalPath(uri); @@ -155,7 +185,7 @@ export async function syncFile(uri) { } } const response = await fetch(remoteURL); - console.log(`downloaded ${uri}`); + console.log(`downloading ${uri}`); await writeFile(localPath, response.body); const lastModified = getLastModifiedDate(response); if (lastModified) { From 4e6f64f3389c9b601408536c2482ef261ec387ec Mon Sep 17 00:00:00 2001 From: YR Chen Date: Wed, 8 Mar 2023 02:18:19 +0800 Subject: [PATCH 09/24] winget-source: support debug mode --- winget-source/sync-repo.js | 5 +++-- winget-source/sync.sh | 4 ++-- winget-source/utilities.js | 7 +++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/winget-source/sync-repo.js b/winget-source/sync-repo.js index f44ac7a..f6e979b 100644 --- a/winget-source/sync-repo.js +++ b/winget-source/sync-repo.js @@ -12,12 +12,13 @@ import { syncFile } from './utilities.js' -const { parallelLimit } = requireEnvironmentVariables(); +const { debugMode, parallelLimit } = requireEnvironmentVariables(); +const { Database } = debugMode ? sqlite3.verbose() : sqlite3; syncFile('source.msix').then(async _ => { const temp = await makeTempDirectory('winget-repo-'); const database = await extractDatabaseFromBundle(getLocalPath('source.msix'), temp); - const db = new sqlite3.Database(database, sqlite3.OPEN_READONLY); + const db = new Database(database, sqlite3.OPEN_READONLY); db.all('SELECT * FROM pathparts', (error, rows) => { const pathparts = buildPathpartMap(error, rows); diff --git a/winget-source/sync.sh b/winget-source/sync.sh index 401cfe2..8a8c392 100644 --- a/winget-source/sync.sh +++ b/winget-source/sync.sh @@ -11,8 +11,8 @@ ## SET IN ENVIRONMENT VARIABLES #DEBUG= -export NODE_ENV=production - set -e +[[ $DEBUG == true ]] && set -x + exec node /sync-repo.js diff --git a/winget-source/utilities.js b/winget-source/utilities.js index 987cc58..e846b53 100644 --- a/winget-source/utilities.js +++ b/winget-source/utilities.js @@ -15,6 +15,9 @@ const local = process.env.TO; /** Maximum sync jobs to be executed in parallel. Defaults to 8. */ const parallelLimit = parseInt(process.env.WINGET_REPO_JOBS ?? 8); +/** Whether the debug mode is enabled. */ +const debugMode = process.env.DEBUG === 'true'; + /** * Get last modified date from HTTP response headers. * @@ -102,11 +105,11 @@ export function buildURIList(error, rows, pathparts) { * @returns Environment variables to be used in the program. */ export function requireEnvironmentVariables() { - if (!remote || !local || !parallelLimit) { + if (!local || !remote) { console.error("required environment variable(s) not set!"); process.exit(-1); } - return { remote, local, parallelLimit }; + return { debugMode, local, parallelLimit, remote }; } /** From a0f60dc2e296d52102e350e875da44a1de628b29 Mon Sep 17 00:00:00 2001 From: YR Chen Date: Wed, 8 Mar 2023 02:20:32 +0800 Subject: [PATCH 10/24] winget-source: skip update if bundle is unchanged --- winget-source/sync-repo.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/winget-source/sync-repo.js b/winget-source/sync-repo.js index f6e979b..33a3a3a 100644 --- a/winget-source/sync-repo.js +++ b/winget-source/sync-repo.js @@ -15,7 +15,12 @@ import { const { debugMode, parallelLimit } = requireEnvironmentVariables(); const { Database } = debugMode ? sqlite3.verbose() : sqlite3; -syncFile('source.msix').then(async _ => { +syncFile('source.msix').then(async updated => { + if (!updated) { + console.info('nothing to update'); + return; + } + const temp = await makeTempDirectory('winget-repo-'); const database = await extractDatabaseFromBundle(getLocalPath('source.msix'), temp); const db = new Database(database, sqlite3.OPEN_READONLY); From 56edf94eada4a9edeb060de5012353a41f816782 Mon Sep 17 00:00:00 2001 From: YR Chen Date: Wed, 8 Mar 2023 02:44:22 +0800 Subject: [PATCH 11/24] winget-source: add description in README --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 4898eb8..7ae3f46 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ - [rsync](#rsync) - [rubygems](#rubygems) - [stackage](#stackage) + - [winget-source](#winget-source) - [yum-sync](#yum-sync) - [License](#license) - [Contributing](#contributing) @@ -413,6 +414,18 @@ Stackage doesn't need to specify upstream, but this mirror use cabal to install Read the [user guide](https://www.haskell.org/cabal/users-guide/installing-packages.html#repository-specification) before writing preferred mirror to `config` +### winget-source + +[![winget-source](https://img.shields.io/docker/image-size/ustcmirror/winget-source/latest)](https://hub.docker.com/r/ustcmirror/winget-source "winget-source") +[![winget-source](https://img.shields.io/docker/pulls/ustcmirror/winget-source)](https://hub.docker.com/r/ustcmirror/winget-source "winget-source") + +A handy tool to sync pre-indexed [Windows Package Manager](https://github.com/microsoft/winget-cli) (aka. WinGet) sources. + +| Parameter | Description | +| --------------------- | ---------------------------------------------------------------- | +| `WINGET_REPO_URL` | Sets the URL of upstream. | +| `WINGET_REPO_JOBS` | Parallel jobs. Defaults to 8. | + ### yum-sync [![yum-sync](https://img.shields.io/docker/image-size/ustcmirror/yum-sync/latest)](https://hub.docker.com/r/ustcmirror/yum-sync "yum-sync") From d022afe1fd15951ac233a90334fa95780a607c8e Mon Sep 17 00:00:00 2001 From: YR Chen Date: Wed, 8 Mar 2023 02:47:33 +0800 Subject: [PATCH 12/24] winget-source: add default upstream URL --- README.md | 2 +- winget-source/utilities.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7ae3f46..5d17f25 100644 --- a/README.md +++ b/README.md @@ -423,7 +423,7 @@ A handy tool to sync pre-indexed [Windows Package Manager](https://github.com/mi | Parameter | Description | | --------------------- | ---------------------------------------------------------------- | -| `WINGET_REPO_URL` | Sets the URL of upstream. | +| `WINGET_REPO_URL` | Sets the URL of upstream. Defaults to [`https://cdn.winget.microsoft.com/cache`](https://cdn.winget.microsoft.com/cache) | | `WINGET_REPO_JOBS` | Parallel jobs. Defaults to 8. | ### yum-sync diff --git a/winget-source/utilities.js b/winget-source/utilities.js index e846b53..b11bc8a 100644 --- a/winget-source/utilities.js +++ b/winget-source/utilities.js @@ -7,7 +7,7 @@ import { existsSync } from 'fs' import { mkdir, mkdtemp, readFile, stat, utimes, writeFile } from 'fs/promises' /** The remote URL of a pre-indexed WinGet source repository. */ -const remote = process.env.WINGET_REPO_URL; +const remote = process.env.WINGET_REPO_URL ?? 'https://cdn.winget.microsoft.com/cache'; /** The local path to serve as the root of WinGet source repository. */ const local = process.env.TO; From ed1c477e5fcb0cbf1b32ee3bc77f6c5c0eddfadf Mon Sep 17 00:00:00 2001 From: YR Chen Date: Wed, 8 Mar 2023 03:06:12 +0800 Subject: [PATCH 13/24] winget-source: fix file permission --- winget-source/sync.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 winget-source/sync.sh diff --git a/winget-source/sync.sh b/winget-source/sync.sh old mode 100644 new mode 100755 From a84568236e4a47b0c6849e0d0b5f99ff8a53e523 Mon Sep 17 00:00:00 2001 From: YR Chen Date: Wed, 8 Mar 2023 04:07:18 +0800 Subject: [PATCH 14/24] winget-source: style fix --- winget-source/Dockerfile | 10 +++++----- winget-source/sync.sh | 5 ++--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/winget-source/Dockerfile b/winget-source/Dockerfile index 2859b70..7bdbccf 100644 --- a/winget-source/Dockerfile +++ b/winget-source/Dockerfile @@ -5,8 +5,8 @@ ADD ["sync.sh", "sync-repo.js", "utilities.js", "package.json", "package-lock.js RUN apk add --no-cache nodejs -RUN cd / \ - && apk add --no-cache npm python3 \ - && npm ci --only=production \ - && npm cache clean --force \ - && apk del --purge --no-cache npm python3 +RUN cd / && \ + apk add --no-cache npm python3 && \ + npm ci --only=production && \ + npm cache clean --force && \ + apk del --purge --no-cache npm python3 diff --git a/winget-source/sync.sh b/winget-source/sync.sh index 8a8c392..165a8e3 100755 --- a/winget-source/sync.sh +++ b/winget-source/sync.sh @@ -1,8 +1,5 @@ #!/bin/bash -#WINGET_REPO_URL= -#WINGET_REPO_JOBS= - ## EXPORTED IN entry.sh #TO= #LOGDIR= @@ -10,6 +7,8 @@ ## SET IN ENVIRONMENT VARIABLES #DEBUG= +#WINGET_REPO_URL= +#WINGET_REPO_JOBS= set -e From 96b834b80c0dcf09c372f59c1123d17f44d547a4 Mon Sep 17 00:00:00 2001 From: YR Chen Date: Wed, 8 Mar 2023 11:27:25 +0800 Subject: [PATCH 15/24] winget-source: support logging to file --- winget-source/package-lock.json | 191 +++++++++++++++++++++++++++++++- winget-source/package.json | 3 +- winget-source/sync-repo.js | 12 +- winget-source/sync.sh | 6 +- winget-source/utilities.js | 69 +++++++++--- 5 files changed, 258 insertions(+), 23 deletions(-) diff --git a/winget-source/package-lock.json b/winget-source/package-lock.json index 274c3cd..798403a 100644 --- a/winget-source/package-lock.json +++ b/winget-source/package-lock.json @@ -12,7 +12,26 @@ "async": "^3.2.4", "jszip": "^3.10.1", "node-fetch": "^3.3.0", - "sqlite3": "^5.1.4" + "sqlite3": "^5.1.4", + "winston": "^3.8.2" + } + }, + "node_modules/@colors/colors": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz", + "integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==", + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/@dabh/diagnostics": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz", + "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==", + "dependencies": { + "colorspace": "1.1.x", + "enabled": "2.0.x", + "kuler": "^2.0.0" } }, "node_modules/@gar/promisify": { @@ -92,6 +111,11 @@ "node": ">= 6" } }, + "node_modules/@types/triple-beam": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.2.tgz", + "integrity": "sha512-txGIh+0eDFzKGC25zORnswy+br1Ha7hj5cMVwKIU7+s0U2AxxJru/jZSMU6OC9MJWP6+pc/hc6ZjyZShpsyY2g==" + }, "node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -225,6 +249,37 @@ "node": ">=6" } }, + "node_modules/color": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", + "integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==", + "dependencies": { + "color-convert": "^1.9.3", + "color-string": "^1.6.0" + } + }, + "node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, "node_modules/color-support": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", @@ -233,6 +288,15 @@ "color-support": "bin.js" } }, + "node_modules/colorspace": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz", + "integrity": "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==", + "dependencies": { + "color": "^3.1.3", + "text-hex": "1.0.x" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -299,6 +363,11 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, + "node_modules/enabled": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", + "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==" + }, "node_modules/encoding": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", @@ -323,6 +392,11 @@ "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==", "optional": true }, + "node_modules/fecha": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", + "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==" + }, "node_modules/fetch-blob": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", @@ -345,6 +419,11 @@ "node": "^12.20 || >= 14.13" } }, + "node_modules/fn.name": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", + "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" + }, "node_modules/formdata-polyfill": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", @@ -523,6 +602,11 @@ "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==", "optional": true }, + "node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" + }, "node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", @@ -537,6 +621,17 @@ "integrity": "sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==", "optional": true }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", @@ -586,6 +681,11 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/kuler": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", + "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" + }, "node_modules/lie": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", @@ -594,6 +694,19 @@ "immediate": "~3.0.5" } }, + "node_modules/logform": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/logform/-/logform-2.5.1.tgz", + "integrity": "sha512-9FyqAm9o9NKKfiAKfZoYo9bGXXuwMkxQiQttkT4YjjVtQVIQtK6LmVtlxmCaFswo6N4AfEkHqZTV0taDtPotNg==", + "dependencies": { + "@colors/colors": "1.5.0", + "@types/triple-beam": "^1.3.2", + "fecha": "^4.2.0", + "ms": "^2.1.1", + "safe-stable-stringify": "^2.3.1", + "triple-beam": "^1.3.0" + } + }, "node_modules/lru-cache": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", @@ -930,6 +1043,14 @@ "wrappy": "1" } }, + "node_modules/one-time": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", + "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", + "dependencies": { + "fn.name": "1.x.x" + } + }, "node_modules/p-map": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", @@ -1037,6 +1158,14 @@ } ] }, + "node_modules/safe-stable-stringify": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.2.tgz", + "integrity": "sha512-gMxvPJYhP0O9n2pvcfYfIuYgbledAOJFcqRThtPRmjscaipiwcwPPKLytpVzMkG2HAN87Qmo2d4PtGiri1dSLA==", + "engines": { + "node": ">=10" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -1072,6 +1201,14 @@ "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, "node_modules/smart-buffer": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", @@ -1144,6 +1281,14 @@ "node": ">= 8" } }, + "node_modules/stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", + "engines": { + "node": "*" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -1200,11 +1345,21 @@ "node": ">=8" } }, + "node_modules/text-hex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", + "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" + }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, + "node_modules/triple-beam": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", + "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==" + }, "node_modules/unique-filename": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz", @@ -1273,6 +1428,40 @@ "string-width": "^1.0.2 || 2 || 3 || 4" } }, + "node_modules/winston": { + "version": "3.8.2", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.8.2.tgz", + "integrity": "sha512-MsE1gRx1m5jdTTO9Ld/vND4krP2To+lgDoMEHGGa4HIlAUyXJtfc7CxQcGXVyz2IBpw5hbFkj2b/AtUdQwyRew==", + "dependencies": { + "@colors/colors": "1.5.0", + "@dabh/diagnostics": "^2.0.2", + "async": "^3.2.3", + "is-stream": "^2.0.0", + "logform": "^2.4.0", + "one-time": "^1.0.0", + "readable-stream": "^3.4.0", + "safe-stable-stringify": "^2.3.1", + "stack-trace": "0.0.x", + "triple-beam": "^1.3.0", + "winston-transport": "^4.5.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/winston-transport": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.5.0.tgz", + "integrity": "sha512-YpZzcUzBedhlTAfJg6vJDlyEai/IFMIVcaEZZyl3UXIl4gmqRpU7AE89AHLkbzLUsv0NVmw7ts+iztqKxxPW1Q==", + "dependencies": { + "logform": "^2.3.2", + "readable-stream": "^3.6.0", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 6.4.0" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/winget-source/package.json b/winget-source/package.json index 4d5c34c..0b99aea 100644 --- a/winget-source/package.json +++ b/winget-source/package.json @@ -10,7 +10,8 @@ "async": "^3.2.4", "jszip": "^3.10.1", "node-fetch": "^3.3.0", - "sqlite3": "^5.1.4" + "sqlite3": "^5.1.4", + "winston": "^3.8.2" }, "scripts": { "start": "node sync-repo.js" diff --git a/winget-source/sync-repo.js b/winget-source/sync-repo.js index 33a3a3a..5079281 100644 --- a/winget-source/sync-repo.js +++ b/winget-source/sync-repo.js @@ -9,15 +9,20 @@ import { getLocalPath, makeTempDirectory, requireEnvironmentVariables, + setupWinstonLogger, syncFile } from './utilities.js' -const { debugMode, parallelLimit } = requireEnvironmentVariables(); +const { debugMode, logFile, parallelLimit, remote } = requireEnvironmentVariables(); + const { Database } = debugMode ? sqlite3.verbose() : sqlite3; +const logger = setupWinstonLogger(debugMode, logFile); + +logger.info(`start syncing with ${remote}`); syncFile('source.msix').then(async updated => { if (!updated) { - console.info('nothing to update'); + logger.info('nothing to update'); return; } @@ -33,9 +38,10 @@ syncFile('source.msix').then(async updated => { async.eachLimit(uris, parallelLimit, syncFile, (error) => { rm(temp, { recursive: true }); if (error) { - console.error(error); + logger.error(error); process.exit(-1); } + logger.info(`successfully synced with ${remote}`); }); }); }); diff --git a/winget-source/sync.sh b/winget-source/sync.sh index 165a8e3..1138758 100755 --- a/winget-source/sync.sh +++ b/winget-source/sync.sh @@ -12,6 +12,10 @@ set -e -[[ $DEBUG == true ]] && set -x +if [[ $DEBUG = true ]]; then + set -x +else + export NODE_ENV=production +fi exec node /sync-repo.js diff --git a/winget-source/utilities.js b/winget-source/utilities.js index b11bc8a..f16c0e7 100644 --- a/winget-source/utilities.js +++ b/winget-source/utilities.js @@ -3,6 +3,7 @@ import fetch from 'node-fetch' import os from 'os' import path from 'path' import process from 'process' +import winston from 'winston' import { existsSync } from 'fs' import { mkdir, mkdtemp, readFile, stat, utimes, writeFile } from 'fs/promises' @@ -18,6 +19,9 @@ const parallelLimit = parseInt(process.env.WINGET_REPO_JOBS ?? 8); /** Whether the debug mode is enabled. */ const debugMode = process.env.DEBUG === 'true'; +/** Path of the log file to write. */ +const logFile = process.env.LOGFILE; + /** * Get last modified date from HTTP response headers. * @@ -74,7 +78,7 @@ function resolvePathpart(id, pathparts) { */ export function buildPathpartMap(error, rows) { if (error) { - console.error(error); + winston.error(error); process.exit(-1); } return new Map(rows.map(row => @@ -93,25 +97,12 @@ export function buildPathpartMap(error, rows) { */ export function buildURIList(error, rows, pathparts) { if (error) { - console.error(error); + winston.error(error); process.exit(-1); } return rows.map(row => resolvePathpart(row.pathpart, pathparts)); } -/** - * Check and return the required environment variables. - * - * @returns Environment variables to be used in the program. - */ -export function requireEnvironmentVariables() { - if (!local || !remote) { - console.error("required environment variable(s) not set!"); - process.exit(-1); - } - return { debugMode, local, parallelLimit, remote }; -} - /** * Extract database file from the source bundle. * @@ -164,6 +155,50 @@ export async function makeTempDirectory(prefix) { return await mkdtemp(path.join(os.tmpdir(), prefix)); } +/** + * Check and return the required environment variables. + * + * @returns Environment variables to be used in the program. + */ +export function requireEnvironmentVariables() { + if (!local || !logFile) { + console.error("[ERROR] required environment variable(s) not set!"); + process.exit(-1); + } + return { debugMode, local, logFile, parallelLimit, remote }; +} + +/** + * Set up the default `winston` logger instance. + * + * @param {boolean} debugMode Whether the debug mode is enabled. + * @param {fs.PathLike} logFile Path of the log file to write. + * + * @returns The shared `winston` logger. + */ +export function setupWinstonLogger(debugMode, logFile) { + const { format, transports } = winston; + const logWithLevelPrefix = ({ level, message }) => `[${level.toUpperCase()}] ${message}`; + winston.configure({ + transports: [ + new transports.File({ + level: 'verbose', + filename: logFile, + format: format.combine( + format.timestamp(), + format.printf((info) => `[${info.timestamp}]${logWithLevelPrefix(info)}`) + ) + }), + new transports.Console({ + level: debugMode ? 'debug' : 'info', + stderrLevels: ['error'], + format: format.printf(logWithLevelPrefix) + }) + ] + }); + return winston; +} + /** * Sync a file with the remote server asynchronously. * @@ -182,13 +217,13 @@ export async function syncFile(uri) { if (lastModified) { const localFile = await stat(localPath); if (localFile.mtime.getTime() == lastModified.getTime() && localFile.size == contentLength) { - console.info(`skipped ${uri} because it's up to date`); + winston.verbose(`skipped ${uri} because it's up to date`); return false; } } } const response = await fetch(remoteURL); - console.log(`downloading ${uri}`); + winston.verbose(`downloading from ${remoteURL}`); await writeFile(localPath, response.body); const lastModified = getLastModifiedDate(response); if (lastModified) { From 8389af86c8f2602fbc8468d8cf1baf1c3cf2291f Mon Sep 17 00:00:00 2001 From: YR Chen Date: Wed, 8 Mar 2023 14:01:02 +0800 Subject: [PATCH 16/24] winget-source: sync newer manifests first --- winget-source/sync-repo.js | 2 +- winget-source/utilities.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/winget-source/sync-repo.js b/winget-source/sync-repo.js index 5079281..0e25adc 100644 --- a/winget-source/sync-repo.js +++ b/winget-source/sync-repo.js @@ -32,7 +32,7 @@ syncFile('source.msix').then(async updated => { db.all('SELECT * FROM pathparts', (error, rows) => { const pathparts = buildPathpartMap(error, rows); - db.all('SELECT pathpart FROM manifest', (error, rows) => { + db.all('SELECT pathpart FROM manifest ORDER BY rowid DESC', (error, rows) => { db.close(); const uris = buildURIList(error, rows, pathparts); async.eachLimit(uris, parallelLimit, syncFile, (error) => { diff --git a/winget-source/utilities.js b/winget-source/utilities.js index f16c0e7..33268b5 100644 --- a/winget-source/utilities.js +++ b/winget-source/utilities.js @@ -222,8 +222,8 @@ export async function syncFile(uri) { } } } - const response = await fetch(remoteURL); winston.verbose(`downloading from ${remoteURL}`); + const response = await fetch(remoteURL); await writeFile(localPath, response.body); const lastModified = getLastModifiedDate(response); if (lastModified) { From 67d11a97f3bfd5adf7272825efeda9693a55bd3d Mon Sep 17 00:00:00 2001 From: YR Chen Date: Wed, 8 Mar 2023 14:33:24 +0800 Subject: [PATCH 17/24] winget-source: update NPM package name --- winget-source/package-lock.json | 4 ++-- winget-source/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/winget-source/package-lock.json b/winget-source/package-lock.json index 798403a..e592a86 100644 --- a/winget-source/package-lock.json +++ b/winget-source/package-lock.json @@ -1,11 +1,11 @@ { - "name": "winget-source", + "name": "@ustcmirror/winget-source", "version": "1.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "winget-source", + "name": "@ustcmirror/winget-source", "version": "1.0.0", "license": "MIT", "dependencies": { diff --git a/winget-source/package.json b/winget-source/package.json index 0b99aea..ca2db74 100644 --- a/winget-source/package.json +++ b/winget-source/package.json @@ -1,5 +1,5 @@ { - "name": "winget-source", + "name": "@ustcmirror/winget-source", "version": "1.0.0", "description": "Sync with pre-indexed WinGet source repository.", "main": "sync-repo.js", From 374550c934f80a682b39e8be061ba54ffd1d7dd5 Mon Sep 17 00:00:00 2001 From: YR Chen Date: Wed, 8 Mar 2023 14:46:10 +0800 Subject: [PATCH 18/24] winget-source: remove skipped messages from log --- winget-source/utilities.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/winget-source/utilities.js b/winget-source/utilities.js index 33268b5..6d96690 100644 --- a/winget-source/utilities.js +++ b/winget-source/utilities.js @@ -217,7 +217,7 @@ export async function syncFile(uri) { if (lastModified) { const localFile = await stat(localPath); if (localFile.mtime.getTime() == lastModified.getTime() && localFile.size == contentLength) { - winston.verbose(`skipped ${uri} because it's up to date`); + winston.debug(`skipped ${uri} because it's up to date`); return false; } } From 25ab342d6a15ba4853f8524bd8899b24e75fdf0b Mon Sep 17 00:00:00 2001 From: YR Chen Date: Wed, 8 Mar 2023 15:54:24 +0800 Subject: [PATCH 19/24] winget-source: simplify setup --- winget-source/sync-repo.js | 19 ++++------ winget-source/utilities.js | 74 +++++++++++++++++--------------------- 2 files changed, 40 insertions(+), 53 deletions(-) diff --git a/winget-source/sync-repo.js b/winget-source/sync-repo.js index 0e25adc..bf82396 100644 --- a/winget-source/sync-repo.js +++ b/winget-source/sync-repo.js @@ -1,5 +1,4 @@ import async from 'async' -import sqlite3 from 'sqlite3' import { rm } from 'fs/promises' import { @@ -8,27 +7,23 @@ import { extractDatabaseFromBundle, getLocalPath, makeTempDirectory, - requireEnvironmentVariables, - setupWinstonLogger, + setupEnvironment, syncFile } from './utilities.js' -const { debugMode, logFile, parallelLimit, remote } = requireEnvironmentVariables(); +const { parallelLimit, remote, sqlite3, winston } = setupEnvironment(); -const { Database } = debugMode ? sqlite3.verbose() : sqlite3; -const logger = setupWinstonLogger(debugMode, logFile); - -logger.info(`start syncing with ${remote}`); +winston.info(`start syncing with ${remote}`); syncFile('source.msix').then(async updated => { if (!updated) { - logger.info('nothing to update'); + winston.info('nothing to update'); return; } const temp = await makeTempDirectory('winget-repo-'); const database = await extractDatabaseFromBundle(getLocalPath('source.msix'), temp); - const db = new Database(database, sqlite3.OPEN_READONLY); + const db = new sqlite3.Database(database, sqlite3.OPEN_READONLY); db.all('SELECT * FROM pathparts', (error, rows) => { const pathparts = buildPathpartMap(error, rows); @@ -38,10 +33,10 @@ syncFile('source.msix').then(async updated => { async.eachLimit(uris, parallelLimit, syncFile, (error) => { rm(temp, { recursive: true }); if (error) { - logger.error(error); + winston.error(error); process.exit(-1); } - logger.info(`successfully synced with ${remote}`); + winston.info(`successfully synced with ${remote}`); }); }); }); diff --git a/winget-source/utilities.js b/winget-source/utilities.js index 6d96690..ae8f58e 100644 --- a/winget-source/utilities.js +++ b/winget-source/utilities.js @@ -3,6 +3,7 @@ import fetch from 'node-fetch' import os from 'os' import path from 'path' import process from 'process' +import sqlite3 from 'sqlite3' import winston from 'winston' import { existsSync } from 'fs' import { mkdir, mkdtemp, readFile, stat, utimes, writeFile } from 'fs/promises' @@ -19,9 +20,6 @@ const parallelLimit = parseInt(process.env.WINGET_REPO_JOBS ?? 8); /** Whether the debug mode is enabled. */ const debugMode = process.env.DEBUG === 'true'; -/** Path of the log file to write. */ -const logFile = process.env.LOGFILE; - /** * Get last modified date from HTTP response headers. * @@ -68,6 +66,23 @@ function resolvePathpart(id, pathparts) { return path.posix.join(resolvePathpart(pathpart.parent, pathparts), pathpart.pathpart); } +/** + * Set up the default `winston` logger instance. + */ +function setupWinstonLogger() { + const { format, transports } = winston; + winston.add(new transports.Console({ + level: debugMode ? 'debug' : 'info', + stderrLevels: ['error'], + format: format.combine( + format.timestamp(), + format.printf(({ timestamp, level, message }) => { + return `[${timestamp}][${level.toUpperCase()}] ${message}`; + }) + ) + })); +} + /** * Build a local storage for path parts from database query. * @@ -156,47 +171,24 @@ export async function makeTempDirectory(prefix) { } /** - * Check and return the required environment variables. + * Check and set up the environment. * - * @returns Environment variables to be used in the program. + * @returns Values and objects to be used in the program. */ -export function requireEnvironmentVariables() { - if (!local || !logFile) { - console.error("[ERROR] required environment variable(s) not set!"); +export function setupEnvironment() { + setupWinstonLogger(); + if (!local) { + winston.error("destination path $TO not set!"); process.exit(-1); } - return { debugMode, local, logFile, parallelLimit, remote }; -} - -/** - * Set up the default `winston` logger instance. - * - * @param {boolean} debugMode Whether the debug mode is enabled. - * @param {fs.PathLike} logFile Path of the log file to write. - * - * @returns The shared `winston` logger. - */ -export function setupWinstonLogger(debugMode, logFile) { - const { format, transports } = winston; - const logWithLevelPrefix = ({ level, message }) => `[${level.toUpperCase()}] ${message}`; - winston.configure({ - transports: [ - new transports.File({ - level: 'verbose', - filename: logFile, - format: format.combine( - format.timestamp(), - format.printf((info) => `[${info.timestamp}]${logWithLevelPrefix(info)}`) - ) - }), - new transports.Console({ - level: debugMode ? 'debug' : 'info', - stderrLevels: ['error'], - format: format.printf(logWithLevelPrefix) - }) - ] - }); - return winston; + return { + debugMode, + local, + parallelLimit, + remote, + sqlite3: debugMode ? sqlite3.verbose() : sqlite3, + winston + }; } /** @@ -222,7 +214,7 @@ export async function syncFile(uri) { } } } - winston.verbose(`downloading from ${remoteURL}`); + winston.info(`downloading from ${remoteURL}`); const response = await fetch(remoteURL); await writeFile(localPath, response.body); const lastModified = getLastModifiedDate(response); From 783fef3aa5fd4736bab0feb11a0df7314bcabc5b Mon Sep 17 00:00:00 2001 From: YR Chen Date: Wed, 8 Mar 2023 16:32:59 +0800 Subject: [PATCH 20/24] winget-source: support `BIND_ADDRESS` --- winget-source/utilities.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/winget-source/utilities.js b/winget-source/utilities.js index ae8f58e..6d43049 100644 --- a/winget-source/utilities.js +++ b/winget-source/utilities.js @@ -1,3 +1,4 @@ +import https from 'https' import JSZip from 'jszip' import fetch from 'node-fetch' import os from 'os' @@ -20,6 +21,9 @@ const parallelLimit = parseInt(process.env.WINGET_REPO_JOBS ?? 8); /** Whether the debug mode is enabled. */ const debugMode = process.env.DEBUG === 'true'; +/** Local IP address to be bound to HTTPS requests. */ +const localAddress = process.env.BIND_ADDRESS; + /** * Get last modified date from HTTP response headers. * @@ -181,6 +185,9 @@ export function setupEnvironment() { winston.error("destination path $TO not set!"); process.exit(-1); } + if (localAddress) { + https.globalAgent.options.localAddress = localAddress; + } return { debugMode, local, From ddcbd8ebd14da664c576c09f97fe2a046f252c43 Mon Sep 17 00:00:00 2001 From: YR Chen Date: Wed, 8 Mar 2023 16:37:41 +0800 Subject: [PATCH 21/24] winget-source: clarify actually used options --- winget-source/sync.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/winget-source/sync.sh b/winget-source/sync.sh index 1138758..4ddb67b 100755 --- a/winget-source/sync.sh +++ b/winget-source/sync.sh @@ -2,10 +2,9 @@ ## EXPORTED IN entry.sh #TO= -#LOGDIR= -#LOGFILE= ## SET IN ENVIRONMENT VARIABLES +#BIND_ADDRESS= #DEBUG= #WINGET_REPO_URL= #WINGET_REPO_JOBS= From 2150373ec3e8b0aae61577a86cdebcc7829251d0 Mon Sep 17 00:00:00 2001 From: YR Chen Date: Thu, 9 Mar 2023 16:15:05 +0800 Subject: [PATCH 22/24] winget-source: simplify `Dockerfile` --- winget-source/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/winget-source/Dockerfile b/winget-source/Dockerfile index 7bdbccf..846e7fb 100644 --- a/winget-source/Dockerfile +++ b/winget-source/Dockerfile @@ -5,8 +5,7 @@ ADD ["sync.sh", "sync-repo.js", "utilities.js", "package.json", "package-lock.js RUN apk add --no-cache nodejs -RUN cd / && \ - apk add --no-cache npm python3 && \ - npm ci --only=production && \ +RUN apk add --no-cache npm python3 && \ + npm ci --prefix / --only=production && \ npm cache clean --force && \ apk del --purge --no-cache npm python3 From f44d7fb0d94b8183a4dbdc9092a092fc89abb662 Mon Sep 17 00:00:00 2001 From: YR Chen Date: Thu, 9 Mar 2023 16:28:42 +0800 Subject: [PATCH 23/24] winget-source: improves Docker image layer cache --- winget-source/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/winget-source/Dockerfile b/winget-source/Dockerfile index 846e7fb..72dd3e6 100644 --- a/winget-source/Dockerfile +++ b/winget-source/Dockerfile @@ -1,11 +1,12 @@ FROM ustcmirror/base:alpine LABEL maintainer "YR Chen " -ADD ["sync.sh", "sync-repo.js", "utilities.js", "package.json", "package-lock.json", "/"] - RUN apk add --no-cache nodejs +ADD ["package.json", "package-lock.json", "/"] RUN apk add --no-cache npm python3 && \ npm ci --prefix / --only=production && \ npm cache clean --force && \ apk del --purge --no-cache npm python3 + +ADD ["sync.sh", "sync-repo.js", "utilities.js", "/"] From a3f3f42d8c131933747602b60deba4f10ff72f82 Mon Sep 17 00:00:00 2001 From: YR Chen Date: Thu, 9 Mar 2023 16:41:34 +0800 Subject: [PATCH 24/24] winget-source: style update --- winget-source/utilities.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/winget-source/utilities.js b/winget-source/utilities.js index 6d43049..3816bd5 100644 --- a/winget-source/utilities.js +++ b/winget-source/utilities.js @@ -62,7 +62,7 @@ function getContentLength(response) { * @param {number} id The ID of the target path part. * @param {Map} pathparts Path part storage built from the database. * - * @returns {string} Full URI resolved from the given ID. + * @returns {string} Full URI resolved from the given path part ID. */ function resolvePathpart(id, pathparts) { const pathpart = pathparts.get(id); @@ -80,9 +80,9 @@ function setupWinstonLogger() { stderrLevels: ['error'], format: format.combine( format.timestamp(), - format.printf(({ timestamp, level, message }) => { - return `[${timestamp}][${level.toUpperCase()}] ${message}`; - }) + format.printf(({ timestamp, level, message }) => + `[${timestamp}][${level.toUpperCase()}] ${message}` + ) ) })); }