From 1f3e7b5bf0593de94640a6a89cd4734f9322dfda Mon Sep 17 00:00:00 2001 From: Shuchit Date: Wed, 13 Nov 2024 16:33:15 +0530 Subject: [PATCH 01/27] ref: replace 'whoami' with 'id -u -n' for POSIX compatibility --- bun/README.md | 2 +- fish/README.md | 4 ++-- go/README.md | 2 +- golang/README.md | 2 +- postgres/README.md | 4 ++-- ssh-adduser/ssh-adduser | 2 +- ssh-pubkey/README.md | 2 +- ssh-pubkey/ssh-pubkey | 8 ++++---- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/bun/README.md b/bun/README.md index 89f0699e7..dd7c74240 100644 --- a/bun/README.md +++ b/bun/README.md @@ -134,7 +134,7 @@ file) ```sh sudo env PATH="$PATH" \ serviceman add --path="$PATH" --system \ - --username "$(whoami)" --name my-project -- \ + --username "$(id -u -n)" --name my-project -- \ bun run ./my-project.js ``` 4. Restart the logging service diff --git a/fish/README.md b/fish/README.md index 2e27699ef..7e672da94 100644 --- a/fish/README.md +++ b/fish/README.md @@ -43,7 +43,7 @@ the file: ```sh #!/bin/bash -echo "Who am I? I'm $(whoami)." +echo "Who am I? I'm $(id -u -n)." ``` You can also run bash explicitly: @@ -99,7 +99,7 @@ You should use `chsh` to change your shell: ```sh #!/bin/sh -sudo chsh -s "$(command -v fish)" "$(whoami)" +sudo chsh -s "$(command -v fish)" "$(id -u -n)" ``` If vim uses `fish` instead of `bash`, annoying errors will happen. diff --git a/go/README.md b/go/README.md index f3a0a5168..68b415a79 100644 --- a/go/README.md +++ b/go/README.md @@ -81,7 +81,7 @@ pushd ./hello/ # swap 'hello' and './hello' for the name of your project and binary sudo env PATH="$PATH" \ - serviceman add --system --username "$(whoami)" --name hello -- \ + serviceman add --system --username "$(id -u -n)" --name hello -- \ ./hello # Restart the logging service diff --git a/golang/README.md b/golang/README.md index 1bc0e0939..8c064775c 100644 --- a/golang/README.md +++ b/golang/README.md @@ -86,7 +86,7 @@ pushd ./hello/ # swap 'hello' and './hello' for the name of your project and binary sudo env PATH="$PATH" \ - serviceman add --system --username "$(whoami)" --name hello -- \ + serviceman add --system --username "$(id -u -n)" --name hello -- \ ./hello # Restart the logging service diff --git a/postgres/README.md b/postgres/README.md index b72250390..6f1e89b25 100644 --- a/postgres/README.md +++ b/postgres/README.md @@ -35,7 +35,7 @@ To enable Postgres as a Linux Service with [serviceman](../serviceman/): \ ```sh sudo env PATH="$PATH" \ - serviceman add --system --username "$(whoami)" --name 'postgres' -- \ + serviceman add --system --username "$(id -u -n)" --name 'postgres' -- \ postgres -D ~/.local/share/postgres/var -p 5432 sudo systemctl restart systemd-journald @@ -120,7 +120,7 @@ curl https://webi.sh/serviceman | sh ```sh sudo env PATH="$PATH" \ - serviceman add --system --username "$(whoami)" --name 'postgres' -- \ + serviceman add --system --username "$(id -u -n)" --name 'postgres' -- \ postgres -D ~/.local/share/postgres/var -p 5432 sudo systemctl restart systemd-journald diff --git a/ssh-adduser/ssh-adduser b/ssh-adduser/ssh-adduser index a9335644f..2fdf1fadc 100644 --- a/ssh-adduser/ssh-adduser +++ b/ssh-adduser/ssh-adduser @@ -14,7 +14,7 @@ main() { my_key_url="${2:-}" my_keys="" - if [ "root" != "$(whoami)" ]; then + if [ "root" != "$(id -u -n)" ]; then echo "webi adduser: running user is already a non-root user" exit 0 fi diff --git a/ssh-pubkey/README.md b/ssh-pubkey/README.md index 84d2e134c..768f073d1 100644 --- a/ssh-pubkey/README.md +++ b/ssh-pubkey/README.md @@ -61,7 +61,7 @@ folder: ```sh rsync -av "$HOME/.ssh/id_rsa.pub" \ - "$HOME/Downloads/id_rsa.$(whoami).pub" + "$HOME/Downloads/id_rsa.$(id -u -n).pub" ``` How to print your public key to the Terminal: diff --git a/ssh-pubkey/ssh-pubkey b/ssh-pubkey/ssh-pubkey index 61ab2281a..f7ace425d 100755 --- a/ssh-pubkey/ssh-pubkey +++ b/ssh-pubkey/ssh-pubkey @@ -82,11 +82,11 @@ main() { # TODO use the comment (if any) for the name of the file echo >&2 "" #shellcheck disable=SC2088 - echo >&2 "~/Downloads/id_${my_keytype}.$(whoami).pub": + echo >&2 "~/Downloads/id_${my_keytype}.$(id -u -n).pub": echo >&2 "" - rm -f "$HOME/Downloads/id_${my_keytype}.$(whoami).pub" - cp -RPp "$HOME/.ssh/id_${my_keytype}.pub" "$HOME/Downloads/id_${my_keytype}.$(whoami).pub" - cat "$HOME/Downloads/id_${my_keytype}.$(whoami).pub" + rm -f "$HOME/Downloads/id_${my_keytype}.$(id -u -n).pub" + cp -RPp "$HOME/.ssh/id_${my_keytype}.pub" "$HOME/Downloads/id_${my_keytype}.$(id -u -n).pub" + cat "$HOME/Downloads/id_${my_keytype}.$(id -u -n).pub" echo >&2 "" if test -f ~/.ssh/id_rsa; then From d6fc5cec9716d31976d4bf52db5d4151d46388ed Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Sun, 15 Dec 2024 06:07:43 +0000 Subject: [PATCH 02/27] doc: remove excess whitespace for (id -u -n) --- caddy/README.md | 8 ++++---- node/README.md | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/caddy/README.md b/caddy/README.md index 0b28d29a2..4d812cff2 100644 --- a/caddy/README.md +++ b/caddy/README.md @@ -819,7 +819,7 @@ To avoid the nitty-gritty details of `launchd` plist files, you can use 2. Use Serviceman to create a _launchd_ plist file ```sh - my_username="$( id -u -n )" + my_username="$(id -u -n)" serviceman add --user --name caddy -- \ caddy run --config ./Caddyfile --envfile ~/.config/caddy/env @@ -901,7 +901,7 @@ See the notes below to run as a **User Service** or use the JSON Config. ``` 4. Use Serviceman to create a _systemd_ config file. ```sh - my_username="$( id -u -n )" + my_username="$(id -u -n)" sudo env PATH="$PATH" \ serviceman add --system --username "${my_username}" --name caddy -- \ caddy run --config ./Caddyfile --envfile ~/.config/caddy/env @@ -1363,7 +1363,7 @@ See also: 2. Generate the `service` file: \ - JSON Config ```sh - my_app_user="$( id -u -n )" + my_app_user="$(id -u -n)" sudo env PATH="${PATH}" \ serviceman add --system --cap-net-bind \ --username "${my_app_user}" --name caddy -- \ @@ -1371,7 +1371,7 @@ See also: ``` - Caddyfile ```sh - my_app_user="$( id -u -n )" + my_app_user="$(id -u -n)" sudo env PATH="${PATH}" \ serviceman add --system --cap-net-bind \ --username "${my_app_user}" --name caddy -- \ diff --git a/node/README.md b/node/README.md index 27a4436ec..59034c1f8 100644 --- a/node/README.md +++ b/node/README.md @@ -227,7 +227,7 @@ Node app as a Non-System (Unprivileged) Service on Mac, Windows, and Linux: or _User Unit_ (Linux): ```sh - my_username="$( id -u -n )" + my_username="$(id -u -n)" serviceman add --user --name my-node-project -- \ caddy run --config ./Caddyfile --envfile ~/.config/caddy/env @@ -275,7 +275,7 @@ Node app as a Non-System (Unprivileged) Service on Mac, Windows, and Linux: ```sh pushd ./my-node-project/ -my_username="$( id -u -n )" +my_username="$(id -u -n)" sudo env PATH="$PATH" \ serviceman add --system --path "$PATH" --cap-net-bind \ --name my-node-project --username "${my_username}" -- \ From 801df24541b8ad0e87bb1e727781423900608f13 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Sun, 15 Dec 2024 06:45:18 +0000 Subject: [PATCH 03/27] chore: remove junk mariadb releases (never completed) --- mariadb/releases.js | 210 -------------------------------------------- 1 file changed, 210 deletions(-) delete mode 100644 mariadb/releases.js diff --git a/mariadb/releases.js b/mariadb/releases.js deleted file mode 100644 index 32d3d0675..000000000 --- a/mariadb/releases.js +++ /dev/null @@ -1,210 +0,0 @@ -'use strict'; - -var brewReleases = require('../_common/brew.js'); - -module.exports = function (request) { - // So many places to get (incomplete) release info... - // - // MariaDB official - // - https://downloads.mariadb.org/mariadb/+releases/ - // - http://archive.mariadb.org/ - // Brew - // - https://formulae.brew.sh/api/formula/mariadb@10.3.json - // - https://formulae.brew.sh/docs/api/ - // - https://formulae.brew.sh/formula/mariadb@10.2#default - // - // Note: This could be very fragile due to using the html - // as an API. It's pretty rather than minified, but that - // doesn't guarantee that it's meant as a consumable API. - // - - var promises = [mariaReleases(), brewReleases(request, 'mariadb')]; - return Promise.all(promises).then(function (many) { - var versions = many[0]; - var brews = many[1]; - - var all = { download: '', releases: [] }; - - // linux x86 - // linux x64 - // windows x86 - // windows x64 - // (and mac, wedged-in from Homebrew) - versions.forEach(function (ver) { - all.releases.push({ - version: ver.version, - lts: false, - channel: ver.channel, - date: ver.date, - os: 'linux', - arch: 'amd64', - download: - 'http://archive.mariadb.org/mariadb-{{ v }}/bintar-linux-x86_64/mariadb-{{ v }}-linux-x86_64.tar.gz'.replace( - /{{ v }}/g, - ver.version, - ), - }); - all.releases.push({ - version: ver.version, - lts: false, - channel: ver.channel, - date: ver.date, - os: 'linux', - arch: 'amd64', - download: - 'http://archive.mariadb.org/mariadb-{{ v }}/bintar-linux-x86/mariadb-{{ v }}-linux-x86.tar.gz'.replace( - /{{ v }}/g, - ver.version, - ), - }); - - // windows - all.releases.push({ - version: ver.version, - lts: false, - channel: ver.channel, - date: ver.date, - os: 'windows', - arch: 'amd64', - download: - 'http://archive.mariadb.org/mariadb-{{ v }}/winx64-packages/mariadb-{{ v }}-winx64.zip'.replace( - /{{ v }}/g, - ver.version, - ), - }); - all.releases.push({ - version: ver.version, - lts: false, - channel: ver.channel, - date: ver.date, - os: 'windows', - arch: 'x86', - download: - 'http://archive.mariadb.org/mariadb-{{ v }}/win32-packages/mariadb-{{ v }}-win32.zip'.replace( - /{{ v }}/g, - ver.version, - ), - }); - - // Note: versions are sorted most-recent first. - // We just assume that the brew version is most recent stable - // ... but we can't really know for sure - - // TODO - brews.some(function (brew, i) { - // 10.3 => ^10.2(\b|\.) - var reBrewVer = new RegExp( - '^' + brew.version.replace(/\./, '\\.') + '(\\b|\\.)', - 'g', - ); - if (!ver.version.match(reBrewVer)) { - return; - } - all.releases.push({ - version: ver.version, - lts: false, - channel: ver.channel, - date: ver.date, - os: 'macos', - arch: 'amd64', - download: brew.download.replace(/{{ v }}/g, ver.version), - }); - brews.splice(i, 1); // remove - return true; - }); - }); - - return all; - }); - - function mariaReleases() { - return request({ - url: 'https://downloads.mariadb.org/mariadb/+releases/', - fail: true, // https://git.coolaj86.com/coolaj86/request.js/issues/2 - }) - .then(failOnBadStatus) - .then(function (resp) { - // fragile, but simple - - // Make release info go from this: - var html = resp.body; - // - // - // 10.0.38 - // 2019-01-31 - // Stable - // - - // To this: - var reLine = /\s*(<(tr|td)[^>]*>)\s*/g; - // - // 10.0.382019-01-31Stable - // 10.0.372018-11-01Stable - // 10.0.362018-08-01Stable - // - // To this: - var reVer = - /.*mariadb\/(10[^\/]+)\/">.*(20\d\d-\d\d-\d\d)<\/td>(\w+)<\/td>/; - // - // { "version": "10.0.36", "date": "2018-08-01", "channel": "stable" } - - return html - .replace(reLine, '$1') - .split(/\n/) - .map(function (line) { - var m = line.match(reVer); - if (!m) { - return; - } - return { - version: m[1], - channel: mapChannel(m[3].toLowerCase()), - date: m[2], - }; - }) - .filter(Boolean); - }) - .catch(function (err) { - console.error('Error fetching (official) MariaDB versions'); - console.error(err); - return []; - }); - } -}; - -function mapChannel(ch) { - if ('alpha' === ch) { - return 'dev'; - } - // stable,rc,beta - return ch; -} - -function failOnBadStatus(resp) { - if (resp.statusCode >= 400) { - var err = new Error('Non-successful status code: ' + resp.statusCode); - err.code = 'ESTATUS'; - err.response = resp; - throw err; - } - return resp; -} - -if (module === require.main) { - module.exports(require('@root/request')).then(function (all) { - console.info('official releases look like:'); - console.info(JSON.stringify(all.releases.slice(0, 2), null, 2)); - console.info('Homebrew releases look like:'); - console.info( - JSON.stringify( - all.releases - .filter(function (rel) { - return 'macos' === rel.os; - }) - .slice(0, 2), - null, - 2, - ), - ); - }); -} From ba94ad883b05e45fc9f285804234d896a4aca03f Mon Sep 17 00:00:00 2001 From: MichalTirpak Date: Wed, 4 Dec 2024 00:06:44 +0100 Subject: [PATCH 04/27] partial refactor for files regarding the ISSUE#898 request to fetch besides mariadb --- flutter/releases.js | 17 ++++++---- go/releases.js | 52 +++++++++++++++--------------- gpg/releases.js | 24 +++++++++++--- iterm2/releases.js | 48 ++++++++++++++++++--------- macos/releases.js | 75 +++++++++++++++++++++++++++---------------- node/releases.js | 46 +++++++++++++++++--------- terraform/releases.js | 27 ++++++++++++---- zig/releases.js | 31 +++++++++++++----- 8 files changed, 208 insertions(+), 112 deletions(-) diff --git a/flutter/releases.js b/flutter/releases.js index f2e097b19..472d446a3 100644 --- a/flutter/releases.js +++ b/flutter/releases.js @@ -53,7 +53,7 @@ var channelMap = {}; // ] // } -module.exports = async function (request) { +module.exports = async function () { let all = { download: '', releases: [], @@ -61,13 +61,17 @@ module.exports = async function (request) { }; for (let osname of FLUTTER_OSES) { - let resp = await request({ - url: `https://storage.googleapis.com/flutter_infra_release/releases/releases_${osname}.json`, - json: true, + const response = await fetch(`https://storage.googleapis.com/flutter_infra_release/releases/releases_${osname}.json`, { + method: 'GET', + headers: { Accept: 'application/json' }, }); + if (!response.ok) { + throw new Error(`Failed to fetch data for ${osname}: ${response.statusText}`); + } + const respBody = await response.json(); - let osBaseUrl = resp.body.base_url; - let osReleases = resp.body.releases; + let osBaseUrl = respBody.base_url; + let osReleases = respBody.releases; for (let asset of osReleases) { if (!channelMap[asset.channel]) { @@ -80,7 +84,6 @@ module.exports = async function (request) { lts: false, channel: asset.channel, date: asset.release_date.replace(/T.*/, ''), - //sha256: asset.sha256, download: `${osBaseUrl}/${asset.archive}`, _filename: asset.archive, }); diff --git a/go/releases.js b/go/releases.js index f712aebbf..715bdbc72 100644 --- a/go/releases.js +++ b/go/releases.js @@ -18,7 +18,7 @@ function isOdd(filename) { } } -function getDistributables(request) { +async function getDistributables() { /* { version: 'go1.13.8', @@ -37,54 +37,54 @@ function getDistributables(request) { ] }; */ - return request({ - url: 'https://golang.org/dl/?mode=json&include=all', - json: true, - }).then((resp) => { - var goReleases = resp.body; - var all = { + const response = await fetch('https://golang.org/dl/?mode=json&include=all', { + method: 'GET', + headers: { Accept: 'application/json' }, + }); + if (!response.ok) { + throw new Error(`Failed to fetch Go releases: ${response.statusText}`); + } + + const goReleases = await response.json(); + const all = { releases: [], download: '', }; goReleases.forEach((release) => { - // strip 'go' prefix, standardize version - var parts = release.version.slice(2).split('.'); + // Strip 'go' prefix and standardize version + const parts = release.version.slice(2).split('.'); while (parts.length < 3) { parts.push('0'); } - var version = parts.join('.'); - // nix 'go' prefix - var fileversion = release.version.slice(2); - + const version = parts.join('.'); + const fileversion = release.version.slice(2); + release.files.forEach((asset) => { - let odd = isOdd(asset.filename); - if (odd) { + if (isOdd(asset.filename)) { return; } - - var filename = asset.filename; - var os = osMap[asset.os] || asset.os || '-'; - var arch = archMap[asset.arch] || asset.arch || '-'; + + const filename = asset.filename; + const os = osMap[asset.os] || asset.os || '-'; + const arch = archMap[asset.arch] || asset.arch || '-'; all.releases.push({ version: version, _version: fileversion, - // all go versions >= 1.0.0 are effectively LTS lts: (parts[0] > 0 && release.stable) || false, channel: (release.stable && 'stable') || 'beta', - date: '1970-01-01', // the world may never know + date: '1970-01-01', // Placeholder os: os, arch: arch, - ext: '', // let normalize run the split/test/join - hash: '-', // not ready to standardize this yet + ext: '', // Let normalize run the split/test/join + hash: '-', // Placeholder for hash download: `https://dl.google.com/go/${filename}`, }); }); }); - + return all; - }); -} + } module.exports = getDistributables; diff --git a/gpg/releases.js b/gpg/releases.js index bea218a8b..ef8e844a8 100644 --- a/gpg/releases.js +++ b/gpg/releases.js @@ -16,15 +16,31 @@ function createUrlMatcher() { ); } -async function getRawReleases(request) { +async function getRawReleases() { let matcher = createRssMatcher(); - let resp = await request({ - url: 'https://sourceforge.net/projects/gpgosx/rss?path=/', + const response = await fetch('https://sourceforge.net/projects/gpgosx/rss?path=/', { + method: 'GET', + headers: { + 'Accept': 'application/rss+xml', // Ensure the correct content type is requested + }, }); + + // Validate the response status + if (!response.ok) { + throw new Error(`Failed to fetch RSS feed: HTTP ${response.status} - ${response.statusText}`); + } + + const contentType = response.headers.get('Content-Type'); + if (!contentType || !contentType.includes('xml')) { + throw new Error(`Unexpected content type: ${contentType}`); + } + + const body = await response.text(); // Fetch RSS feed as plain text + let links = []; for (;;) { - let m = matcher.exec(resp.body); + let m = matcher.exec(body); if (!m) { break; } diff --git a/iterm2/releases.js b/iterm2/releases.js index 26e3167c8..6d5503541 100644 --- a/iterm2/releases.js +++ b/iterm2/releases.js @@ -1,22 +1,38 @@ 'use strict'; -function getRawReleases(request) { - return request({ url: 'https://iterm2.com/downloads.html' }).then( - function (resp) { - var links = resp.body - .split(/[<>]+/g) - .map(function (str) { - var m = str.match( - /href="(https:\/\/iterm2\.com\/downloads\/.*\.zip)"/, - ); - if (m && /iTerm2-[34]/.test(m[1])) { - return m[1]; - } - }) - .filter(Boolean); - return links; +async function getRawReleases() { + const response = await fetch('https://iterm2.com/downloads.html', { + method: 'GET', + headers: { + 'Accept': 'text/html', // Explicitly request HTML content }, - ); + }); + + // Validate HTTP response + if (!response.ok) { + throw new Error(`Failed to fetch releases: HTTP ${response.status} - ${response.statusText}`); + } + + // Validate Content-Type header + const contentType = response.headers.get('Content-Type'); + if (!contentType || !contentType.includes('text/html')) { + throw new Error(`Unexpected Content-Type: ${contentType}`); + } + + // Parse HTML content + const body = await response.text(); + var links = body + .split(/[<>]+/g) + .map(function (str) { + var m = str.match( + /href="(https:\/\/iterm2\.com\/downloads\/.*\.zip)"/, + ); + if (m && /iTerm2-[34]/.test(m[1])) { + return m[1]; + } + }) + .filter(Boolean); + return links; } function transformReleases(links) { diff --git a/macos/releases.js b/macos/releases.js index ae443b033..10d0ad231 100644 --- a/macos/releases.js +++ b/macos/releases.js @@ -40,23 +40,41 @@ var headers = { 'Accept-Language': 'en-US,en;q=0.9,sq;q=0.8', }; -module.exports = function (request) { - var all = { +async function fetchReleasesForOS(os) { + // Fetch the webpage for the given OS + const response = await fetch(os.url, { + method: 'GET', + headers: headers, + }); + + // Validate HTTP response + if (!response.ok) { + throw new Error(`Failed to fetch URL: ${os.url}. HTTP ${response.status} - ${response.statusText}`); + } + + // Parse the response body + const body = await response.text(); + + // Extract the download link + const match = body.match(/(http[^>]+Install[^>]+\.dmg)/); + return match ? match[1] : null; +} + +async function getDistributables() { + const all = { _names: ['InstallOS'], download: '', releases: [], }; - return Promise.all( - oses.map(function (os) { - return request({ - method: 'GET', - url: os.url, - headers: headers, - }).then(function (resp) { - var m = resp.body.match(/(http[^>]+Install[^>]+.dmg)/); - var download = m && m[1]; - ['macos', 'linux'].forEach(function (osname) { + // Fetch data for each OS and populate the releases array + await Promise.all( + oses.map(async (os) => { + try { + const download = await fetchReleasesForOS(os); + + // Add releases for macOS and Linux + ['macos', 'linux'].forEach((osname) => { all.releases.push({ version: os.version, lts: os.lts || false, @@ -65,27 +83,28 @@ module.exports = function (request) { os: osname, arch: 'amd64', ext: 'dmg', - hash: '-', + hash: '-', // Placeholder for hash download: download, }); }); - }); - }), - ).then(function () { - all.releases.sort(function (a, b) { - if ('10.11.6' === a.version) { - return -1; + } catch (err) { + console.error(`Error fetching for ${os.name}: ${err.message}`); } - if (a.date > b.date) { - return 1; - } - if (a.date < b.date) { - return -1; - } - }); - return all; + }), + ); + + // Sort releases + all.releases.sort((a, b) => { + if (a.version === '10.11.6') { + return -1; + } + return a.date > b.date ? 1 : -1; }); -}; + + return all; +} + +module.exports = getDistributables; if (module === require.main) { module.exports(require('@root/request')).then(function (all) { diff --git a/node/releases.js b/node/releases.js index ac896018e..00a57c85f 100644 --- a/node/releases.js +++ b/node/releases.js @@ -40,7 +40,7 @@ let pkgMap = { musl: ['tar.gz', 'tar.xz'], }; -async function getDistributables(request) { +async function getDistributables() { let all = { releases: [], download: '', @@ -64,26 +64,40 @@ async function getDistributables(request) { ] */ - // Alternate: 'https://nodejs.org/dist/index.json', - let baseUrl = `https://nodejs.org/download/release`; - let officialP = request({ - url: `${baseUrl}/index.json`, - json: true, - }).then(function (resp) { - transform(baseUrl, resp.body); - return; + // Alternate: 'https://nodejs.org/dist/index.json', + let baseUrl = `https://nodejs.org/download/release`; + + // Fetch official builds + let officialP = fetch(`${baseUrl}/index.json`, { + method: 'GET', + headers: { Accept: 'application/json' }, + }).then((response) => { + if (!response.ok) { + throw new Error(`Failed to fetch official builds: HTTP ${response.status} - ${response.statusText}`); + } + return response.json(); + }) + .then((data) => { + transform(baseUrl, data); }); + + // Fetch unofficial builds let unofficialBaseUrl = `https://unofficial-builds.nodejs.org/download/release`; - let unofficialP = request({ - url: `${unofficialBaseUrl}/index.json`, - json: true, + let unofficialP = fetch(`${unofficialBaseUrl}/index.json`, { + method: 'GET', + headers: { Accept: 'application/json' }, }) - .then(function (resp) { - transform(unofficialBaseUrl, resp.body); - return; + .then((response) => { + if (!response.ok) { + throw new Error(`Failed to fetch unofficial builds: HTTP ${response.status} - ${response.statusText}`); + } + return response.json(); + }) + .then((data) => { + transform(unofficialBaseUrl, data); }) - .catch(function (err) { + .catch((err) => { console.error('failed to fetch unofficial-builds'); console.error(err); }); diff --git a/terraform/releases.js b/terraform/releases.js index d473bcd7a..1928d9e18 100644 --- a/terraform/releases.js +++ b/terraform/releases.js @@ -1,11 +1,21 @@ 'use strict'; -function getDistributables(request) { - return request({ - url: 'https://releases.hashicorp.com/terraform/index.json', - json: true, - }).then(function (resp) { - let releases = resp.body; +async function getDistributables() { + try { + // Fetch the Terraform releases JSON + const response = await fetch('https://releases.hashicorp.com/terraform/index.json', { + method: 'GET', + headers: { Accept: 'application/json' }, + }); + + // Validate the HTTP response + if (!response.ok) { + throw new Error(`Failed to fetch releases: HTTP ${response.status} - ${response.statusText}`); + } + + // Parse the JSON response + const releases = await response.json(); + let all = { releases: [], download: '', // Full URI provided in response body @@ -34,7 +44,10 @@ function getDistributables(request) { }); return all; - }); + } catch (err) { + console.error('Error fetching Terraform releases:', err.message); + return { releases: [], download: '' }; + } } module.exports = getDistributables; diff --git a/zig/releases.js b/zig/releases.js index 3d0dff30f..4c447eb9d 100644 --- a/zig/releases.js +++ b/zig/releases.js @@ -3,12 +3,22 @@ var NON_BUILDS = ['bootstrap', 'src']; var ODDITIES = NON_BUILDS.concat(['armv6kz-linux']); -module.exports = function (request) { - return request({ - url: 'https://ziglang.org/download/index.json', - json: true, - }).then(function (resp) { - let versions = resp.body; +module.exports = async function () { + try { + // Fetch the Zig language download index JSON + const response = await fetch('https://ziglang.org/download/index.json', { + method: 'GET', + headers: { Accept: 'application/json' }, + }); + + // Validate HTTP response + if (!response.ok) { + throw new Error(`Failed to fetch releases: HTTP ${response.status} - ${response.statusText}`); + } + + // Parse the JSON response + const versions = await response.json(); + let releases = []; let refs = Object.keys(versions); @@ -77,8 +87,13 @@ module.exports = function (request) { return { releases: releases, }; - }); -}; + }catch (err) { + console.error('Error fetching Zig releases:', err.message); + return { + releases: [], + }; + }; +} if (module === require.main) { module.exports(require('@root/request')).then(function (all) { From 14cebeeb61d3890c61aba5d36552551d43530566 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Sun, 15 Dec 2024 06:48:52 +0000 Subject: [PATCH 05/27] ref(webi): complete transition from 'request' for 'fetch' --- _common/gitea.js | 8 +- _webi/builds-cacher.js | 4 +- _webi/classify-one.js | 4 +- _webi/transform-releases.js | 3 +- flutter/releases.js | 39 ++++++--- go/releases.js | 113 ++++++++++++++---------- gpg/releases.js | 107 +++++++++++++---------- iterm2/releases.js | 109 +++++++++++------------ macos/releases.js | 105 ++++++++++++++--------- node/releases.js | 108 +++++++++++++---------- terraform/releases.js | 84 +++++++++--------- zig/releases.js | 166 ++++++++++++++++++------------------ 12 files changed, 465 insertions(+), 385 deletions(-) diff --git a/_common/gitea.js b/_common/gitea.js index 51b0e811e..3789d933d 100644 --- a/_common/gitea.js +++ b/_common/gitea.js @@ -41,9 +41,7 @@ if (module === require.main) { 'https://git.rootprojects.org', '', '', - ).then( - function (all) { - console.info(JSON.stringify(all, null, 2)); - }, - ); + ).then(function (all) { + console.info(JSON.stringify(all, null, 2)); + }); } diff --git a/_webi/builds-cacher.js b/_webi/builds-cacher.js index a644ef215..b2780e6d5 100644 --- a/_webi/builds-cacher.js +++ b/_webi/builds-cacher.js @@ -9,8 +9,6 @@ let HostTargets = require('./build-classifier/host-targets.js'); let Lexver = require('./build-classifier/lexver.js'); let Triplet = require('./build-classifier/triplet.js'); -let request = require('@root/request'); - var ALIAS_RE = /^alias: (\w+)$/m; var LEGACY_ARCH_MAP = { @@ -153,7 +151,7 @@ async function getLatestBuilds(Releases, installersDir, cacheDir, name, date) { } async function getLatestBuildsInner(Releases, cacheDir, name, date) { - let data = await Releases.latest(request); + let data = await Releases.latest(); if (!date) { date = new Date(); diff --git a/_webi/classify-one.js b/_webi/classify-one.js index 561274aca..7a1d376de 100644 --- a/_webi/classify-one.js +++ b/_webi/classify-one.js @@ -6,8 +6,6 @@ let Path = require('node:path'); let BuildsCacher = require('./builds-cacher.js'); let Triplet = require('./build-classifier/triplet.js'); -let request = require('@root/request'); - async function main() { let projName = process.argv[2]; if (!projName) { @@ -47,7 +45,7 @@ async function main() { Releases.latest = Releases; } - let projInfo = await Releases.latest(request); + let projInfo = await Releases.latest(); // let packages = await Builds.getPackage({ name: projName }); // console.log(packages); diff --git a/_webi/transform-releases.js b/_webi/transform-releases.js index 050c9ebe8..d6e33d447 100644 --- a/_webi/transform-releases.js +++ b/_webi/transform-releases.js @@ -3,7 +3,6 @@ var Releases = module.exports; var path = require('path'); -var request = require('@root/request'); var _normalize = require('./normalize.js'); var cache = {}; @@ -28,7 +27,7 @@ Releases.get = async function (pkgdir) { throw err; } - let all = await get.latest(request); + let all = await get.latest(); return _normalize(all); }; diff --git a/flutter/releases.js b/flutter/releases.js index 472d446a3..f985b444b 100644 --- a/flutter/releases.js +++ b/flutter/releases.js @@ -2,7 +2,10 @@ var FLUTTER_OSES = ['macos', 'linux', 'windows']; -// stable, beta, dev +/** + * stable, beta, dev + * @type {Object.} + */ var channelMap = {}; // This can be spot-checked against @@ -53,25 +56,39 @@ var channelMap = {}; // ] // } +/** + * @typedef BuildInfo + * @prop {String} version + * @prop {String} [_version] + * @prop {Boolean} lts + * @prop {String} channel + * @prop {String} date + * @prop {String} download + * @prop {String} [_filename] + */ + module.exports = async function () { let all = { download: '', + /** @type {Array} */ releases: [], + /** @type {Array} */ channels: [], }; for (let osname of FLUTTER_OSES) { - const response = await fetch(`https://storage.googleapis.com/flutter_infra_release/releases/releases_${osname}.json`, { - method: 'GET', - headers: { Accept: 'application/json' }, - }); + let response = await fetch( + `https://storage.googleapis.com/flutter_infra_release/releases/releases_${osname}.json`, + { headers: { Accept: 'application/json' } }, + ); if (!response.ok) { - throw new Error(`Failed to fetch data for ${osname}: ${response.statusText}`); + throw new Error( + `Failed to fetch data for ${osname}: ${response.statusText}`, + ); } - const respBody = await response.json(); - - let osBaseUrl = respBody.base_url; - let osReleases = respBody.releases; + let data = await response.json(); + let osBaseUrl = data.base_url; + let osReleases = data.releases; for (let asset of osReleases) { if (!channelMap[asset.channel]) { @@ -100,7 +117,7 @@ module.exports = async function () { }; if (module === require.main) { - module.exports(require('@root/request')).then(function (all) { + module.exports().then(function (all) { all.releases = all.releases.slice(25); console.info(JSON.stringify(all, null, 2)); }); diff --git a/go/releases.js b/go/releases.js index 715bdbc72..c88aecb08 100644 --- a/go/releases.js +++ b/go/releases.js @@ -1,14 +1,19 @@ 'use strict'; +/** @type {Object.} */ var osMap = { darwin: 'macos', }; +/** @type {Object.} */ var archMap = { 386: 'x86', }; let ODDITIES = ['bootstrap', '-arm6.']; +/** + * @param {String} filename + */ function isOdd(filename) { for (let oddity of ODDITIES) { let isOddity = filename.includes(oddity); @@ -18,6 +23,21 @@ function isOdd(filename) { } } +/** + * @typedef BuildInfo + * @prop {String} version + * @prop {String} [_version] + * @prop {String} arch + * @prop {String} channel + * @prop {String} date + * @prop {String} download + * @prop {String} ext + * @prop {String} [_filename] + * @prop {String} hash + * @prop {Boolean} lts + * @prop {String} os + */ + async function getDistributables() { /* { @@ -37,60 +57,63 @@ async function getDistributables() { ] }; */ - const response = await fetch('https://golang.org/dl/?mode=json&include=all', { - method: 'GET', - headers: { Accept: 'application/json' }, - }); - if (!response.ok) { - throw new Error(`Failed to fetch Go releases: ${response.statusText}`); + let response = await fetch('https://golang.org/dl/?mode=json&include=all', { + method: 'GET', + headers: { Accept: 'application/json' }, + }); + if (!response.ok) { + throw new Error(`Failed to fetch Go releases: ${response.statusText}`); + } + + let goReleases = await response.json(); + let all = { + /** @type {Array} */ + releases: [], + download: '', + }; + + for (let release of goReleases) { + // Strip 'go' prefix, standardize version + let parts = release.version.slice(2).split('.'); + while (parts.length < 3) { + parts.push('0'); } - - const goReleases = await response.json(); - const all = { - releases: [], - download: '', - }; + let version = parts.join('.'); + let fileversion = release.version.slice(2); - goReleases.forEach((release) => { - // Strip 'go' prefix and standardize version - const parts = release.version.slice(2).split('.'); - while (parts.length < 3) { - parts.push('0'); + for (let asset of release.files) { + if (isOdd(asset.filename)) { + continue; } - const version = parts.join('.'); - const fileversion = release.version.slice(2); - - release.files.forEach((asset) => { - if (isOdd(asset.filename)) { - return; - } - - const filename = asset.filename; - const os = osMap[asset.os] || asset.os || '-'; - const arch = archMap[asset.arch] || asset.arch || '-'; - all.releases.push({ - version: version, - _version: fileversion, - lts: (parts[0] > 0 && release.stable) || false, - channel: (release.stable && 'stable') || 'beta', - date: '1970-01-01', // Placeholder - os: os, - arch: arch, - ext: '', // Let normalize run the split/test/join - hash: '-', // Placeholder for hash - download: `https://dl.google.com/go/${filename}`, - }); - }); - }); - - return all; + + let filename = asset.filename; + let os = osMap[asset.os] || asset.os || '-'; + let arch = archMap[asset.arch] || asset.arch || '-'; + let build = { + version: version, + _version: fileversion, + lts: (parts[0] > 0 && release.stable) || false, + channel: (release.stable && 'stable') || 'beta', + date: '1970-01-01', // the world may never know + os: os, + arch: arch, + ext: '', // let normalize run the split/test/join + hash: '-', // not ready to standardize this yet + download: `https://dl.google.com/go/${filename}`, + }; + all.releases.push(build); + } } + return all; +} + module.exports = getDistributables; if (module === require.main) { - getDistributables(require('@root/request')).then(function (all) { + getDistributables().then(function (all) { all = require('../_webi/normalize.js')(all); + //@ts-expect-error all.releases = all.releases.slice(0, 10); console.info(JSON.stringify(all, null, 2)); }); diff --git a/gpg/releases.js b/gpg/releases.js index ef8e844a8..3794773c2 100644 --- a/gpg/releases.js +++ b/gpg/releases.js @@ -16,84 +16,97 @@ function createUrlMatcher() { ); } +/** + * @typedef BuildInfo + * @prop {String} version + * @prop {String} [_version] + * @prop {String} arch + * @prop {String} channel + * @prop {String} date + * @prop {String} download + * @prop {String} ext + * @prop {String} [_filename] + * @prop {String} hash + * @prop {Boolean} lts + * @prop {String} os + */ + async function getRawReleases() { let matcher = createRssMatcher(); - const response = await fetch('https://sourceforge.net/projects/gpgosx/rss?path=/', { - method: 'GET', - headers: { - 'Accept': 'application/rss+xml', // Ensure the correct content type is requested - }, + let resp = await fetch('https://sourceforge.net/projects/gpgosx/rss?path=/', { + headers: { Accept: 'application/rss+xml' }, }); - - // Validate the response status - if (!response.ok) { - throw new Error(`Failed to fetch RSS feed: HTTP ${response.status} - ${response.statusText}`); + let text = await resp.text(); // Fetch RSS feed as plain text + if (!resp.ok) { + throw new Error(`Failed to fetch RSS feed: HTTP ${resp.status}: ${text}`); } - const contentType = response.headers.get('Content-Type'); + let contentType = resp.headers.get('Content-Type'); if (!contentType || !contentType.includes('xml')) { throw new Error(`Unexpected content type: ${contentType}`); } - const body = await response.text(); // Fetch RSS feed as plain text - let links = []; for (;;) { - let m = matcher.exec(body); + let m = matcher.exec(text); if (!m) { break; } links.push(m[1]); } + return links; } +/** + * @param {Array} links + */ function transformReleases(links) { //console.log(JSON.stringify(links, null, 2)); //console.log(links.length); let matcher = createUrlMatcher(); - let releases = links - .map(function (link) { - let isLts = ltsRe.test(link); - let parts = link.match(matcher); - if (!parts || !parts[2]) { - return null; - } - let segs = parts[2].split('.'); - let version = segs.slice(0, 3).join('.'); - if (segs.length > 3) { - version += '+' + segs.slice(3); - } - let fileversion = segs.join('.'); - - return { - name: parts[1], - version: version, - _version: fileversion, - // all go versions >= 1.0.0 are effectively LTS - lts: isLts, - channel: 'stable', - // TODO Sat, 19 Nov 2016 16:17:33 UT - date: '1970-01-01', // the world may never know - os: 'macos', - arch: 'amd64', - ext: 'dmg', - download: link, - }; - }) - .filter(Boolean); + let builds = []; + for (let link of links) { + let isLts = ltsRe.test(link); + let parts = link.match(matcher); + if (!parts || !parts[2]) { + continue; + } + + let segs = parts[2].split('.'); + let version = segs.slice(0, 3).join('.'); + if (segs.length > 3) { + version += '+' + segs.slice(3); + } + let fileversion = segs.join('.'); + + let build = { + name: parts[1], + version: version, + _version: fileversion, + lts: isLts, + channel: 'stable', + // TODO Sat, 19 Nov 2016 16:17:33 UT + date: '1970-01-01', // the world may never know + os: 'macos', + arch: 'amd64', + ext: 'dmg', + download: link, + }; + builds.push(build); + } return { _names: ['GnuPG', 'gpgosx'], - releases: releases, + releases: builds, }; } -async function getDistributables(request) { - let releases = await getRawReleases(request); +async function getDistributables() { + let releases = await getRawReleases(); let all = transformReleases(releases); return all; } @@ -101,7 +114,7 @@ async function getDistributables(request) { module.exports = getDistributables; if (module === require.main) { - getDistributables(require('@root/request')).then(function (all) { + getDistributables().then(function (all) { all = require('../_webi/normalize.js')(all); all.releases = all.releases.slice(0, 10000); console.info(JSON.stringify(all, null, 2)); diff --git a/iterm2/releases.js b/iterm2/releases.js index 6d5503541..2c24d8f85 100644 --- a/iterm2/releases.js +++ b/iterm2/releases.js @@ -1,89 +1,82 @@ 'use strict'; async function getRawReleases() { - const response = await fetch('https://iterm2.com/downloads.html', { - method: 'GET', - headers: { - 'Accept': 'text/html', // Explicitly request HTML content - }, + let resp = await fetch('https://iterm2.com/downloads.html', { + headers: { Accept: 'text/html' }, }); - - // Validate HTTP response - if (!response.ok) { - throw new Error(`Failed to fetch releases: HTTP ${response.status} - ${response.statusText}`); + let text = await resp.text(); + if (!resp.ok) { + throw new Error(`Failed to fetch releases: HTTP ${resp.status}: ${text}`); } - // Validate Content-Type header - const contentType = response.headers.get('Content-Type'); + let contentType = resp.headers.get('Content-Type'); if (!contentType || !contentType.includes('text/html')) { throw new Error(`Unexpected Content-Type: ${contentType}`); } - // Parse HTML content - const body = await response.text(); - var links = body - .split(/[<>]+/g) - .map(function (str) { - var m = str.match( - /href="(https:\/\/iterm2\.com\/downloads\/.*\.zip)"/, - ); + let lines = text.split(/[<>]+/g); + + /** @type {Array} */ + let links = []; + for (let str of lines) { + var m = str.match(/href="(https:\/\/iterm2\.com\/downloads\/.*\.zip)"/); if (m && /iTerm2-[34]/.test(m[1])) { - return m[1]; + if (m[1]) { + links.push(m[1]); + } } - }) - .filter(Boolean); + } + return links; } +/** + * @param {Array} links + */ function transformReleases(links) { - //console.log(JSON.stringify(links, null, 2)); - //console.log(links.length); + let builds = []; + for (let link of links) { + var channel = /\/stable\//.test(link) ? 'stable' : 'beta'; - return { - _names: ['iTerm2', 'iterm2'], - releases: links - .map(function (link) { - var channel = /\/stable\//.test(link) ? 'stable' : 'beta'; + var parts = link.replace(/.*\/iTerm2[-_]v?(\d_.*)\.zip/, '$1').split('_'); + var version = parts.join('.').replace(/([_-])?beta/, '-beta'); - var parts = link - .replace(/.*\/iTerm2[-_]v?(\d_.*)\.zip/, '$1') - .split('_'); - var version = parts.join('.').replace(/([_-])?beta/, '-beta'); + // ex: 3.5.0-beta17 => 3_5_0beta17 + // ex: 3.0.2-preview => 3_0_2-preview + let fileversion = version.replace(/\./g, '_'); + fileversion = fileversion.replace(/-beta/g, 'beta'); - // ex: 3.5.0-beta17 => 3_5_0beta17 - // ex: 3.0.2-preview => 3_0_2-preview - let fileversion = version.replace(/\./g, '_'); - fileversion = fileversion.replace(/-beta/g, 'beta'); + let build = { + version: version, + _version: fileversion, + lts: 'stable' === channel, + channel: channel, + date: '1970-01-01', // the world may never know + os: 'macos', + arch: 'amd64', + ext: '', // let normalize run the split/test/join + download: link, + }; + builds.push(build); + } - return { - version: version, - _version: fileversion, - // all go versions >= 1.0.0 are effectively LTS - lts: 'stable' === channel, - channel: channel, - date: '1970-01-01', // the world may never know - os: 'macos', - arch: 'amd64', - ext: '', // let normalize run the split/test/join - download: link, - }; - }) - .filter(Boolean), + return { + _names: ['iTerm2', 'iterm2'], + releases: builds, }; } -function getDistributables(request) { - return getRawReleases(request) - .then(transformReleases) - .then(function (all) { - return all; - }); +async function getDistributables() { + let rawReleases = await getRawReleases(); + let all = transformReleases(rawReleases); + + return all; } module.exports = getDistributables; if (module === require.main) { - getDistributables(require('@root/request')).then(function (all) { + getDistributables().then(function (all) { all = require('../_webi/normalize.js')(all); all.releases = all.releases.slice(0, 10000); console.info(JSON.stringify(all, null, 2)); diff --git a/macos/releases.js b/macos/releases.js index 10d0ad231..3bc9294ad 100644 --- a/macos/releases.js +++ b/macos/releases.js @@ -40,65 +40,90 @@ var headers = { 'Accept-Language': 'en-US,en;q=0.9,sq;q=0.8', }; +/** + * @param {typeof oses[0]} os + */ async function fetchReleasesForOS(os) { - // Fetch the webpage for the given OS - const response = await fetch(os.url, { - method: 'GET', + let resp = await fetch(os.url, { headers: headers, }); - - // Validate HTTP response - if (!response.ok) { - throw new Error(`Failed to fetch URL: ${os.url}. HTTP ${response.status} - ${response.statusText}`); + let text = await resp.text(); + if (!resp.ok) { + throw new Error( + `Failed to fetch URL: ${os.url}. HTTP ${resp.status}: ${text}`, + ); } - // Parse the response body - const body = await response.text(); - // Extract the download link - const match = body.match(/(http[^>]+Install[^>]+\.dmg)/); - return match ? match[1] : null; + let match = text.match(/(http[^>]+Install[^>]+\.dmg)/); + if (match) { + return match[1]; + } } +/** + * @typedef BuildInfo + * @prop {String} version + * @prop {String} [_version] + * @prop {String} arch + * @prop {String} channel + * @prop {String} date + * @prop {String} download + * @prop {String} ext + * @prop {String} [_filename] + * @prop {String} hash + * @prop {Boolean} lts + * @prop {String} os + */ + +let osnames = ['macos', 'linux']; + async function getDistributables() { - const all = { + let all = { _names: ['InstallOS'], download: '', + /** @type {Array} */ releases: [], }; // Fetch data for each OS and populate the releases array - await Promise.all( - oses.map(async (os) => { - try { - const download = await fetchReleasesForOS(os); - - // Add releases for macOS and Linux - ['macos', 'linux'].forEach((osname) => { - all.releases.push({ - version: os.version, - lts: os.lts || false, - channel: os.channel || 'beta', - date: os.date, - os: osname, - arch: 'amd64', - ext: 'dmg', - hash: '-', // Placeholder for hash - download: download, - }); - }); - } catch (err) { - console.error(`Error fetching for ${os.name}: ${err.message}`); - } - }), - ); + for (let os of oses) { + let download = await fetchReleasesForOS(os); + if (!download) { + continue; + } + + // Add releases for macOS and Linux + for (let osname of osnames) { + let build = { + version: os.version, + lts: os.lts || false, + channel: os.channel || 'beta', + date: os.date, + os: osname, + arch: 'amd64', + ext: 'dmg', + hash: '-', + download: download, + }; + + all.releases.push(build); + } + } // Sort releases - all.releases.sort((a, b) => { + all.releases.sort(function (a, b) { if (a.version === '10.11.6') { return -1; } - return a.date > b.date ? 1 : -1; + + if (a.date > b.date) { + return 1; + } else if (a.date < b.date) { + return -1; + } + + return 0; }); return all; @@ -107,7 +132,7 @@ async function getDistributables() { module.exports = getDistributables; if (module === require.main) { - module.exports(require('@root/request')).then(function (all) { + module.exports().then(function (all) { console.info(JSON.stringify(all, null, 2)); }); } diff --git a/node/releases.js b/node/releases.js index 00a57c85f..3e4fdcc6d 100644 --- a/node/releases.js +++ b/node/releases.js @@ -7,6 +7,7 @@ const END_OF_LIFE = 366 * 24 * 60 * 60 * 1000; // OSes +/** @type {Object.} */ let osMap = { osx: 'macos', // NOTE: filename is 'darwin' linux: 'linux', @@ -16,6 +17,7 @@ let osMap = { }; // CPU architectures +/** @type {Object.} */ let archMap = { x64: 'amd64', x86: 'x86', @@ -28,6 +30,7 @@ let archMap = { }; // file extensions +/** @type {Object.>} */ let pkgMap = { pkg: ['pkg'], //exe: ['exe'], // disable @@ -40,8 +43,25 @@ let pkgMap = { musl: ['tar.gz', 'tar.xz'], }; +/** + * @typedef BuildInfo + * @prop {String} version + * @prop {String} [_version] + * @prop {String} arch + * @prop {String} channel + * @prop {String} date + * @prop {String} download + * @prop {String} ext + * @prop {String} [_filename] + * @prop {String} [hash] + * @prop {String} libc + * @prop {Boolean} lts + * @prop {String} os + */ + async function getDistributables() { let all = { + /** @type {Array} */ releases: [], download: '', }; @@ -64,51 +84,52 @@ async function getDistributables() { ] */ - // Alternate: 'https://nodejs.org/dist/index.json', - let baseUrl = `https://nodejs.org/download/release`; + { + // Alternate: 'https://nodejs.org/dist/index.json', + let baseUrl = `https://nodejs.org/download/release`; - // Fetch official builds - let officialP = fetch(`${baseUrl}/index.json`, { - method: 'GET', - headers: { Accept: 'application/json' }, - }).then((response) => { - if (!response.ok) { - throw new Error(`Failed to fetch official builds: HTTP ${response.status} - ${response.statusText}`); + // Fetch official builds + let resp = await fetch(`${baseUrl}/index.json`, { + headers: { Accept: 'application/json' }, + }); + let text = await resp.text(); + if (!resp.ok) { + throw new Error( + `Failed to fetch official builds: HTTP ${resp.status}: ${text}`, + ); } - return response.json(); - }) - .then((data) => { - transform(baseUrl, data); - }); + let data = JSON.parse(text); + void transform(baseUrl, data); + } - // Fetch unofficial builds - let unofficialBaseUrl = `https://unofficial-builds.nodejs.org/download/release`; - let unofficialP = fetch(`${unofficialBaseUrl}/index.json`, { - method: 'GET', - headers: { Accept: 'application/json' }, - }) - .then((response) => { - if (!response.ok) { - throw new Error(`Failed to fetch unofficial builds: HTTP ${response.status} - ${response.statusText}`); - } - return response.json(); - }) - .then((data) => { - transform(unofficialBaseUrl, data); - }) - .catch((err) => { - console.error('failed to fetch unofficial-builds'); - console.error(err); + { + // Fetch unofficial builds + let unofficialBaseUrl = `https://unofficial-builds.nodejs.org/download/release`; + let resp = await fetch(`${unofficialBaseUrl}/index.json`, { + headers: { Accept: 'application/json' }, }); + let text = await resp.text(); + if (!resp.ok) { + throw new Error( + `Failed to fetch official builds: HTTP ${resp.status}: ${text}`, + ); + } + let data = JSON.parse(text); + transform(unofficialBaseUrl, data); + } + /** + * @param {String} baseUrl + * @param {Array} builds + */ function transform(baseUrl, builds) { - builds.forEach(function (build) { + for (let build of builds) { let buildDate = new Date(build.date).valueOf(); let age = Date.now() - buildDate; let maintained = age < END_OF_LIFE; if (!maintained) { - return; + continue; } let lts = false !== build.lts; @@ -122,9 +143,9 @@ async function getDistributables() { channel = 'beta'; } - build.files.forEach(function (file) { + for (let file of build.files) { if ('src' === file || 'headers' === file) { - return; + continue; } let fileParts = file.split('-'); @@ -140,7 +161,7 @@ async function getDistributables() { pkgs = pkgMap.tar; } if (!pkgs?.length) { - return; + continue; } let extra = ''; @@ -157,7 +178,7 @@ async function getDistributables() { osPart = 'darwin'; } - pkgs.forEach(function (pkg) { + for (let pkg of pkgs) { let filename = `node-${build.version}-${osPart}-${archPart}${extra}.${pkg}`; if ('msi' === pkg) { filename = `node-${build.version}-${archPart}${extra}.${pkg}`; @@ -178,20 +199,17 @@ async function getDistributables() { }; all.releases.push(release); - }); - }); - }); + } + } + } } - await officialP; - await unofficialP; - return all; } module.exports = getDistributables; if (module === require.main) { - getDistributables(require('@root/request')).then(function (all) { + getDistributables().then(function (all) { all = require('../_webi/normalize.js')(all); console.info(JSON.stringify(all)); //console.info(JSON.stringify(all, null, 2)); diff --git a/terraform/releases.js b/terraform/releases.js index 1928d9e18..19c3c1fd7 100644 --- a/terraform/releases.js +++ b/terraform/releases.js @@ -1,59 +1,53 @@ 'use strict'; +/** + * @typedef BuildInfo + * @prop {String} version + * @prop {String} download + */ + async function getDistributables() { - try { - // Fetch the Terraform releases JSON - const response = await fetch('https://releases.hashicorp.com/terraform/index.json', { - method: 'GET', - headers: { Accept: 'application/json' }, - }); - - // Validate the HTTP response - if (!response.ok) { - throw new Error(`Failed to fetch releases: HTTP ${response.status} - ${response.statusText}`); - } + let resp = await fetch( + 'https://releases.hashicorp.com/terraform/index.json', + { headers: { Accept: 'application/json' } }, + ); + let text = await resp.text(); + if (!resp.ok) { + throw new Error(`Failed to fetch releases: HTTP ${resp.status}: ${text}`); + } - // Parse the JSON response - const releases = await response.json(); - - let all = { - releases: [], - download: '', // Full URI provided in response body - }; - - function getBuildsForVersion(version) { - releases.versions[version].builds.forEach(function (build) { - let r = { - version: build.version, - download: build.url, - // These are generic enough for the autodetect, - // and the per-file logic has proven to get outdated sooner - //os: convert[build.os], - //arch: convert[build.arch], - //channel: 'stable|-rc|-beta|-alpha', - }; - all.releases.push(r); - }); + let releases = JSON.parse(text); + let all = { + /** @type {Array} */ + releases: [], + download: '', + }; + + let allVersions = Object.keys(releases.versions); + allVersions.reverse(); // Releases are listed chronologically, we want the latest first. + + for (let version of allVersions) { + for (let build of releases.versions[version].builds) { + let r = { + version: build.version, + download: build.url, + // These are generic enough for the autodetect, + // and the per-file logic has proven to get outdated sooner + //os: convert[build.os], + //arch: convert[build.arch], + //channel: 'stable|-rc|-beta|-alpha', + }; + all.releases.push(r); } - - // Releases are listed chronologically, we want the latest first. - const allVersions = Object.keys(releases.versions).reverse(); - - allVersions.forEach(function (version) { - getBuildsForVersion(version); - }); - - return all; - } catch (err) { - console.error('Error fetching Terraform releases:', err.message); - return { releases: [], download: '' }; } + + return all; } module.exports = getDistributables; if (module === require.main) { - getDistributables(require('@root/request')).then(function (all) { + getDistributables().then(function (all) { all = require('../_webi/normalize.js')(all); console.info(JSON.stringify(all)); }); diff --git a/zig/releases.js b/zig/releases.js index 4c447eb9d..b6fc27c65 100644 --- a/zig/releases.js +++ b/zig/releases.js @@ -3,100 +3,104 @@ var NON_BUILDS = ['bootstrap', 'src']; var ODDITIES = NON_BUILDS.concat(['armv6kz-linux']); -module.exports = async function () { - try { - // Fetch the Zig language download index JSON - const response = await fetch('https://ziglang.org/download/index.json', { - method: 'GET', - headers: { Accept: 'application/json' }, - }); - - // Validate HTTP response - if (!response.ok) { - throw new Error(`Failed to fetch releases: HTTP ${response.status} - ${response.statusText}`); - } - - // Parse the JSON response - const versions = await response.json(); +/** + * @typedef BuildInfo + * @prop {String} version + * @prop {String} [_version] + * @prop {String} [arch] + * @prop {String} channel + * @prop {String} date + * @prop {String} download + * @prop {String} [ext] + * @prop {String} [_filename] + * @prop {String} [hash] + * @prop {String} [libc] + * @prop {Boolean} [lts] + * @prop {String} [size] + * @prop {String} os + */ - let releases = []; +module.exports = async function () { + let resp = await fetch('https://ziglang.org/download/index.json', { + method: 'GET', + headers: { Accept: 'application/json' }, + }); + let text = await resp.text(); + if (!resp.ok) { + throw new Error(`Failed to fetch releases: HTTP ${resp.status}: ${text}`); + } + let versions = JSON.parse(text); - let refs = Object.keys(versions); - refs.forEach(function (ref) { - let pkgs = versions[ref]; - let version = pkgs.version || ref; + /** @type {Array} */ + let releases = []; + let refs = Object.keys(versions); + for (let ref of refs) { + let pkgs = versions[ref]; + let version = pkgs.version || ref; - // "platform" = arch + os combo - let platforms = Object.keys(pkgs); - platforms.forEach(function (platform) { - let pkg = pkgs[platform]; + // "platform" = arch + os combo + let platforms = Object.keys(pkgs); + for (let platform of platforms) { + let pkg = pkgs[platform]; - // don't grab 'date' or 'notes', which are (confusingly) - // at the same level as platform releases - let isNotPackage = !pkg || 'object' !== typeof pkg || !pkg.tarball; - if (isNotPackage) { - return; - } + // don't grab 'date' or 'notes', which are (confusingly) + // at the same level as platform releases + let isNotPackage = !pkg || 'object' !== typeof pkg || !pkg.tarball; + if (isNotPackage) { + continue; + } - let isOdd = ODDITIES.includes(platform); - if (isOdd) { - return; - } + let isOdd = ODDITIES.includes(platform); + if (isOdd) { + continue; + } - // Ex: aarch64-macos => ['aarch64', 'macos'] - let parts = platform.split('-'); - //let arch = parts[0]; - let os = parts[1]; - if (parts.length > 2) { - console.warn( - `unexpected platform name with multiple '-': ${platform}`, - ); - return; - } + // Ex: aarch64-macos => ['aarch64', 'macos'] + let parts = platform.split('-'); + //let arch = parts[0]; + let os = parts[1]; + if (parts.length > 2) { + console.warn(`unexpected platform name with multiple '-': ${platform}`); + continue; + } - let p = { - version: version, - date: pkgs.date, - channel: 'stable', - // linux, macos, windows - os: os, - // TODO map explicitly (rather than normalization auto-detect) - //arch: arch, - download: pkg.tarball, - hash: pkg.shasum, - size: pkg.size, - // TODO docs + release notes? - //docs: 'https://ziglang.org/documentation/0.9.1/', - //stdDocs: 'https://ziglang.org/documentation/0.9.1/std/', - //notes: 'https://ziglang.org/download/0.9.1/release-notes.html' - }; + let p = { + version: version, + date: pkgs.date, + channel: 'stable', + // linux, macos, windows + os: os, + // TODO map explicitly (rather than normalization auto-detect) + //arch: arch, + download: pkg.tarball, + hash: pkg.shasum, + size: pkg.size, + // TODO docs + release notes? + //docs: 'https://ziglang.org/documentation/0.9.1/', + //stdDocs: 'https://ziglang.org/documentation/0.9.1/std/', + //notes: 'https://ziglang.org/download/0.9.1/release-notes.html' + }; - // Mark branches or tags as beta (for now) - // Ex: 'master' - // Also mark prereleases (with build tags) as beta - // Ex: 0.10.0-dev.1606+97a53bb8a - let isNotStable = !/\./.test(ref) || /\+|-/.test(p.version); - if (isNotStable) { - p.channel = 'beta'; - } + // Mark branches or tags as beta (for now) + // Ex: 'master' + // Also mark prereleases (with build tags) as beta + // Ex: 0.10.0-dev.1606+97a53bb8a + let isNotStable = !/\./.test(ref) || /\+|-/.test(p.version); + if (isNotStable) { + p.channel = 'beta'; + } - releases.push(p); - }); - }); + releases.push(p); + } + } - return { - releases: releases, - }; - }catch (err) { - console.error('Error fetching Zig releases:', err.message); - return { - releases: [], - }; + return { + releases: releases, }; -} +}; if (module === require.main) { - module.exports(require('@root/request')).then(function (all) { + module.exports().then(function (all) { all = require('../_webi/normalize.js')(all); // just select the first 5 for demonstration all.releases = all.releases.slice(0, 5); From 217d61ed3486cf069b43770a52778c9110612a7b Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Sun, 15 Dec 2024 06:50:38 +0000 Subject: [PATCH 06/27] doc: remove references to 'request' --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d42fb3cec..71250a610 100644 --- a/README.md +++ b/README.md @@ -145,8 +145,8 @@ It looks like this: `releases.js`: ```js -module.exports = function (request) { - return github(request, owner, repo).then(function (all) { +module.exports = function () { + return github(null, owner, repo).then(function (all) { // if you need to do something special, you can do it here // ... return all; From a5ed5dbe91b5260e2408cf2fee33a1e4e95ad2a5 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Sun, 15 Dec 2024 08:28:36 +0000 Subject: [PATCH 07/27] chore: remove @root/request dependency --- package-lock.json | 6 ------ package.json | 1 - 2 files changed, 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index c5866861b..5a9948c2e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,16 +9,10 @@ "version": "1.1.1", "license": "MPL-2.0", "dependencies": { - "@root/request": "^1.9.2", "dotenv": "^8.2.0", "marked": "^4.1.1" } }, - "node_modules/@root/request": { - "version": "1.9.2", - "resolved": "https://registry.npmjs.org/@root/request/-/request-1.9.2.tgz", - "integrity": "sha512-wVaL9yVV9oDR9UNbPZa20qgY+4Ch6YN8JUkaE4el/uuS5dmhD8Lusm/ku8qJVNtmQA56XLzEDCRS6/vfpiHK2A==" - }, "node_modules/dotenv": { "version": "8.2.0", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz", diff --git a/package.json b/package.json index c5fac7956..dcb1241eb 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,6 @@ }, "homepage": "https://github.com/webinstall/webi-installers#readme", "dependencies": { - "@root/request": "^1.9.2", "dotenv": "^8.2.0", "marked": "^4.1.1" } From fe59a2f35c6d30bc7b5d0831e95da5d6e0e8f967 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Sun, 15 Dec 2024 08:32:35 +0000 Subject: [PATCH 08/27] chore: update deps --- package-lock.json | 25 +++++++++++++++---------- package.json | 4 ++-- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5a9948c2e..2b6fbe908 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,27 +9,32 @@ "version": "1.1.1", "license": "MPL-2.0", "dependencies": { - "dotenv": "^8.2.0", - "marked": "^4.1.1" + "dotenv": "^16.4.7", + "marked": "^15.0.4" } }, "node_modules/dotenv": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz", - "integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==", + "version": "16.4.7", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz", + "integrity": "sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ==", + "license": "BSD-2-Clause", "engines": { - "node": ">=8" + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" } }, "node_modules/marked": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/marked/-/marked-4.1.1.tgz", - "integrity": "sha512-0cNMnTcUJPxbA6uWmCmjWz4NJRe/0Xfk2NhXCUHjew9qJzFN20krFnsUe7QynwqOwa5m1fZ4UDg0ycKFVC0ccw==", + "version": "15.0.4", + "resolved": "https://registry.npmjs.org/marked/-/marked-15.0.4.tgz", + "integrity": "sha512-TCHvDqmb3ZJ4PWG7VEGVgtefA5/euFmsIhxtD0XsBxI39gUSKL81mIRFdt0AiNQozUahd4ke98ZdirExd/vSEw==", + "license": "MIT", "bin": { "marked": "bin/marked.js" }, "engines": { - "node": ">= 12" + "node": ">= 18" } } } diff --git a/package.json b/package.json index dcb1241eb..796061cda 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ }, "homepage": "https://github.com/webinstall/webi-installers#readme", "dependencies": { - "dotenv": "^8.2.0", - "marked": "^4.1.1" + "dotenv": "^16.4.7", + "marked": "^15.0.4" } } From f1d1027701046c9af5ab96dd7483dc31a88add13 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Mon, 16 Dec 2024 00:00:22 +0000 Subject: [PATCH 09/27] ref: handle fetch errors consistently (Fetcher.fetch) --- _common/brew.js | 75 ++++++++++++++++++++----------------- _common/fetcher.js | 56 +++++++++++++++++++++++++++ _common/githubish-source.js | 57 +++++++++++++++++++--------- _common/githubish.js | 35 +++++++++-------- chromedriver/releases.js | 25 ++++++++----- flutter/releases.js | 30 +++++++++------ go/releases.js | 28 +++++++++----- gpg/releases.js | 29 ++++++++------ iterm2/releases.js | 31 +++++++++------ julia/releases.js | 58 ++++++++++++++++++++++------ macos/releases.js | 28 ++++++++------ node/releases.js | 52 +++++++++++++++---------- terraform/releases.js | 24 ++++++++---- zig/releases.js | 28 +++++++++----- 14 files changed, 375 insertions(+), 181 deletions(-) create mode 100644 _common/fetcher.js diff --git a/_common/brew.js b/_common/brew.js index cbe3ac507..04d7a1431 100644 --- a/_common/brew.js +++ b/_common/brew.js @@ -1,49 +1,56 @@ 'use strict'; +let Fetcher = require('../_common/fetcher.js'); + /** * Gets releases from 'brew'. * * @param {null} _ * @param {string} formula - * @returns {PromiseLike | Promise} + * @returns {Promise} */ -function getDistributables(_, formula) { +async function getDistributables(_, formula) { if (!formula) { return Promise.reject('missing formula for brew'); } - return fetch('https://formulae.brew.sh/api/formula/' + formula + '.json') - .then(function (resp) { - if (!resp.ok) { - throw new Error(`HTTP error! Status: ${resp.status}`); - } - return resp.json(); // Parse JSON response - }) - .then(function (body) { - var ver = body.versions.stable; - var dl = ( - body.bottle.stable.files.high_sierra || - body.bottle.stable.files.catalina - ).url.replace(new RegExp(ver.replace(/\./g, '\\.'), 'g'), '{{ v }}'); - return [ - { - version: ver, - download: dl.replace(/{{ v }}/g, ver), - }, - ].concat( - body.versioned_formulae.map(function (f) { - var ver = f.replace(/.*@/, ''); - return { - version: ver, - download: dl, - }; - }), - ); - }) - .catch(function (err) { - console.error('Error fetching MariaDB versions (brew)'); - console.error(err); - return []; + + let resp; + try { + let url = `https://formulae.brew.sh/api/formula/${formula}.json`; + resp = await Fetcher.fetch(url, { + headers: { Accept: 'application/json' }, }); + } catch (e) { + /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error + let err = e; + if (err.code === 'E_FETCH_RELEASES') { + err.message = `failed to fetch '${formula}' release data from 'brew': ${err.response.status} ${err.response.body}`; + } + throw e; + } + let body = JSON.parse(resp.body); + + var ver = body.versions.stable; + var dl = ( + body.bottle.stable.files.high_sierra || body.bottle.stable.files.catalina + ).url.replace(new RegExp(ver.replace(/\./g, '\\.'), 'g'), '{{ v }}'); + return [ + { + version: ver, + download: dl.replace(/{{ v }}/g, ver), + }, + ].concat( + body.versioned_formulae.map( + /** @param {String} f */ + function (f) { + var ver = f.replace(/.*@/, ''); + return { + version: ver, + download: dl, + }; + }, + ), + ); } module.exports = getDistributables; diff --git a/_common/fetcher.js b/_common/fetcher.js new file mode 100644 index 000000000..5158c01c6 --- /dev/null +++ b/_common/fetcher.js @@ -0,0 +1,56 @@ +'use strict'; + +let Fetcher = module.exports; + +/** + * @typedef ResponseSummary + * @prop {Boolean} ok + * @prop {Headers} headers + * @prop {Number} status + * @prop {String} body + */ + +/** + * @param {String} url + * @param {RequestInit} opts + * @returns {Promise} + */ +Fetcher.fetch = async function (url, opts) { + let resp = await fetch(url, opts); + let summary = Fetcher.throwIfNotOk(resp); + + return summary; +}; + +/** + * @param {Response} resp + * @returns {Promise} + */ +Fetcher.throwIfNotOk = async function (resp) { + let text = await resp.text(); + + if (!resp.ok) { + let headers = Array.from(resp.headers); + console.error('[Fetcher] error: Response Headers:', headers); + console.error('[Fetcher] error: Response Text:', text); + let err = new Error(`fetch was not ok`); + Object.assign({ + status: 503, + code: 'E_FETCH_RELEASES', + response: { + status: resp.status, + headers: headers, + body: text, + }, + }); + throw err; + } + + let summary = { + ok: resp.ok, + headers: resp.headers, + status: resp.status, + body: text, + }; + return summary; +}; diff --git a/_common/githubish-source.js b/_common/githubish-source.js index 753fa4285..343ad9284 100644 --- a/_common/githubish-source.js +++ b/_common/githubish-source.js @@ -1,5 +1,7 @@ 'use strict'; +let Fetcher = require('../_common/fetcher.js'); + let GitHubishSource = module.exports; /** @@ -44,30 +46,22 @@ GitHubishSource.getDistributables = async function ({ }); } - let resp = await fetch(url, opts); - if (!resp.ok) { - let headers = Array.from(resp.headers); - console.error('Bad Resp Headers:', headers); - let text = await resp.text(); - console.error('Bad Resp Body:', text); - let msg = `failed to fetch releases from '${baseurl}' with user '${username}'`; - throw new Error(msg); - } - - let respText = await resp.text(); - let gHubResp; + let resp; try { - gHubResp = JSON.parse(respText); + resp = await Fetcher.fetch(url, opts); } catch (e) { - console.error('Bad Resp JSON:', respText); - console.error(e.message); - let msg = `failed to parse releases from '${baseurl}' with user '${username}'`; - throw new Error(msg); + /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error + let err = e; + if (err.code === 'E_FETCH_RELEASES') { + err.message = `failed to fetch '${baseurl}' (githubish-source, user '${username}) release data: ${err.response.status} ${err.response.body}`; + } + throw e; } + let gHubResp = JSON.parse(resp.body); let all = { + /** @type {Array} */ releases: [], - // TODO make this ':baseurl' + ':releasename' download: '', }; @@ -84,6 +78,29 @@ GitHubishSource.getDistributables = async function ({ return all; }; +/** + * @typedef BuildInfo + * @prop {String} [name] - name to use instead of filename for hash urls + * @prop {String} version + * @prop {String} [_version] + * @prop {String} [arch] + * @prop {String} channel + * @prop {String} date + * @prop {String} download + * @prop {String} [ext] + * @prop {String} [_filename] + * @prop {String} [hash] + * @prop {String} [libc] + * @prop {Boolean} [_musl] + * @prop {Boolean} [lts] + * @prop {String} [size] + * @prop {String} os + */ + +/** + * @param {any} ghRelease - TODO + * @returns {Array} + */ GitHubishSource.releaseToDistributables = function (ghRelease) { let ghTag = ghRelease['tag_name']; // TODO tags aren't always semver / sensical let lts = /(\b|_)(lts)(\b|_)/.test(ghRelease['tag_name']); @@ -95,6 +112,7 @@ GitHubishSource.releaseToDistributables = function (ghRelease) { date = date.replace(/T.*/, ''); let urls = [ghRelease.tarball_url, ghRelease.zipball_url]; + /** @type {Array} */ let dists = []; for (let url of urls) { dists.push({ @@ -114,6 +132,9 @@ GitHubishSource.releaseToDistributables = function (ghRelease) { return dists; }; +/** + * @param {BuildInfo} dist + */ GitHubishSource.followDistributableDownloadAttachment = async function (dist) { let abortCtrl = new AbortController(); let resp = await fetch(dist.download, { diff --git a/_common/githubish.js b/_common/githubish.js index ce305a1b3..eedf2deb2 100644 --- a/_common/githubish.js +++ b/_common/githubish.js @@ -1,5 +1,7 @@ 'use strict'; +let Fetcher = require('../_common/fetcher.js'); + /** * @typedef DistributableRaw * @prop {String} name @@ -57,26 +59,18 @@ GitHubish.getDistributables = async function ({ }); } - let resp = await fetch(url, opts); - if (!resp.ok) { - let headers = Array.from(resp.headers); - console.error('Bad Resp Headers:', headers); - let text = await resp.text(); - console.error('Bad Resp Body:', text); - let msg = `failed to fetch releases from '${baseurl}' with user '${username}'`; - throw new Error(msg); - } - - let respText = await resp.text(); - let gHubResp; + let resp; try { - gHubResp = JSON.parse(respText); + resp = await Fetcher.fetch(url, opts); } catch (e) { - console.error('Bad Resp JSON:', respText); - console.error(e.message); - let msg = `failed to parse releases from '${baseurl}' with user '${username}'`; - throw new Error(msg); + /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error + let err = e; + if (err.code === 'E_FETCH_RELEASES') { + err.message = `failed to fetch '${baseurl}' (githubish, user '${username}) release data: ${err.response.status} ${err.response.body}`; + } + throw e; } + let gHubResp = JSON.parse(resp.body); let all = { /** @type {Array} */ @@ -88,13 +82,18 @@ GitHubish.getDistributables = async function ({ try { gHubResp.forEach(transformReleases); } catch (e) { - console.error(e.message); + /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error + let err = e; + console.error(err.message); console.error('Error Headers:', resp.headers); console.error('Error Body:', resp.body); let msg = `failed to transform releases from '${baseurl}' with user '${username}'`; throw new Error(msg); } + /** + * @param {any} release - TODO + */ function transformReleases(release) { for (let asset of release['assets']) { let name = asset['name']; diff --git a/chromedriver/releases.js b/chromedriver/releases.js index 333129f16..ae6b054a7 100644 --- a/chromedriver/releases.js +++ b/chromedriver/releases.js @@ -1,5 +1,7 @@ 'use strict'; +let Fetcher = require('../_common/fetcher.js'); + // See const releaseApiUrl = 'https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json'; @@ -41,18 +43,23 @@ const releaseApiUrl = // } module.exports = async function () { - let resp = await fetch(releaseApiUrl); - - if (!resp.ok) { - let text = await resp.text(); - let msg = `failed to fetch releases from '${releaseApiUrl}': ${resp.status} ${text}`; - throw new Error(msg); + let resp; + try { + resp = await Fetcher.fetch(releaseApiUrl, { + headers: { Accept: 'application/json' }, + }); + } catch (e) { + /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error + let err = e; + if (err.code === 'E_FETCH_RELEASES') { + err.message = `failed to fetch 'chromedriver' release data: ${err.response.status} ${err.response.body}`; + } + throw e; } - - let body = await resp.json(); + let data = JSON.parse(resp.body); let builds = []; - for (let release of body.versions) { + for (let release of data.versions) { if (!release.downloads.chromedriver) { continue; } diff --git a/flutter/releases.js b/flutter/releases.js index f985b444b..49bcb0a23 100644 --- a/flutter/releases.js +++ b/flutter/releases.js @@ -1,12 +1,14 @@ 'use strict'; -var FLUTTER_OSES = ['macos', 'linux', 'windows']; +let Fetcher = require('../_common/fetcher.js'); + +let FLUTTER_OSES = ['macos', 'linux', 'windows']; /** * stable, beta, dev * @type {Object.} */ -var channelMap = {}; +let channelMap = {}; // This can be spot-checked against // https://docs.flutter.dev/release/archive?tab=windows @@ -77,16 +79,22 @@ module.exports = async function () { }; for (let osname of FLUTTER_OSES) { - let response = await fetch( - `https://storage.googleapis.com/flutter_infra_release/releases/releases_${osname}.json`, - { headers: { Accept: 'application/json' } }, - ); - if (!response.ok) { - throw new Error( - `Failed to fetch data for ${osname}: ${response.statusText}`, - ); + let resp; + try { + let url = `https://storage.googleapis.com/flutter_infra_release/releases/releases_${osname}.json`; + resp = await Fetcher.fetch(url, { + headers: { Accept: 'application/json' }, + }); + } catch (e) { + /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error + let err = e; + if (err.code === 'E_FETCH_RELEASES') { + err.message = `failed to fetch 'flutter' release data for ${osname}: ${err.response.status} ${err.response.body}`; + } + throw e; } - let data = await response.json(); + let data = JSON.parse(resp.body); + let osBaseUrl = data.base_url; let osReleases = data.releases; diff --git a/go/releases.js b/go/releases.js index c88aecb08..6e1f3a60e 100644 --- a/go/releases.js +++ b/go/releases.js @@ -1,11 +1,13 @@ 'use strict'; +let Fetcher = require('../_common/fetcher.js'); + /** @type {Object.} */ -var osMap = { +let osMap = { darwin: 'macos', }; /** @type {Object.} */ -var archMap = { +let archMap = { 386: 'x86', }; @@ -57,15 +59,23 @@ async function getDistributables() { ] }; */ - let response = await fetch('https://golang.org/dl/?mode=json&include=all', { - method: 'GET', - headers: { Accept: 'application/json' }, - }); - if (!response.ok) { - throw new Error(`Failed to fetch Go releases: ${response.statusText}`); + + let resp; + try { + let url = 'https://golang.org/dl/?mode=json&include=all'; + resp = await Fetcher.fetch(url, { + headers: { Accept: 'application/json' }, + }); + } catch (e) { + /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error + let err = e; + if (err.code === 'E_FETCH_RELEASES') { + err.message = `failed to fetch 'Go' release data: ${err.response.status} ${err.response.body}`; + } + throw e; } + let goReleases = JSON.parse(resp.body); - let goReleases = await response.json(); let all = { /** @type {Array} */ releases: [], diff --git a/gpg/releases.js b/gpg/releases.js index 3794773c2..8d145839e 100644 --- a/gpg/releases.js +++ b/gpg/releases.js @@ -1,5 +1,7 @@ 'use strict'; +let Fetcher = require('../_common/fetcher.js'); + let ltsRe = /GnuPG-(2\.2\.[\d\.]+)/; function createRssMatcher() { @@ -32,24 +34,29 @@ function createUrlMatcher() { */ async function getRawReleases() { - let matcher = createRssMatcher(); - - let resp = await fetch('https://sourceforge.net/projects/gpgosx/rss?path=/', { - headers: { Accept: 'application/rss+xml' }, - }); - let text = await resp.text(); // Fetch RSS feed as plain text - if (!resp.ok) { - throw new Error(`Failed to fetch RSS feed: HTTP ${resp.status}: ${text}`); + let resp; + try { + let url = 'https://sourceforge.net/projects/gpgosx/rss?path=/'; + resp = await Fetcher.fetch(url, { + headers: { Accept: 'application/rss+xml' }, + }); + } catch (e) { + /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error + let err = e; + if (err.code === 'E_FETCH_RELEASES') { + err.message = `failed to fetch 'gpg' release data: ${err.response.status} ${err.response.body}`; + } + throw e; } - let contentType = resp.headers.get('Content-Type'); - if (!contentType || !contentType.includes('xml')) { + if (!contentType?.includes('xml')) { throw new Error(`Unexpected content type: ${contentType}`); } + let matcher = createRssMatcher(); let links = []; for (;;) { - let m = matcher.exec(text); + let m = matcher.exec(resp.body); if (!m) { break; } diff --git a/iterm2/releases.js b/iterm2/releases.js index 2c24d8f85..f3d61c3e2 100644 --- a/iterm2/releases.js +++ b/iterm2/releases.js @@ -1,12 +1,21 @@ 'use strict'; +let Fetcher = require('../_common/fetcher.js'); + async function getRawReleases() { - let resp = await fetch('https://iterm2.com/downloads.html', { - headers: { Accept: 'text/html' }, - }); - let text = await resp.text(); - if (!resp.ok) { - throw new Error(`Failed to fetch releases: HTTP ${resp.status}: ${text}`); + let resp; + try { + let url = 'https://iterm2.com/downloads.html'; + resp = await Fetcher.fetch(url, { + headers: { Accept: 'text/html' }, + }); + } catch (e) { + /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error + let err = e; + if (err.code === 'E_FETCH_RELEASES') { + err.message = `failed to fetch 'iterm2' release data: ${err.response.status} ${err.response.body}`; + } + throw e; } let contentType = resp.headers.get('Content-Type'); @@ -14,12 +23,12 @@ async function getRawReleases() { throw new Error(`Unexpected Content-Type: ${contentType}`); } - let lines = text.split(/[<>]+/g); + let lines = resp.body.split(/[<>]+/g); /** @type {Array} */ let links = []; for (let str of lines) { - var m = str.match(/href="(https:\/\/iterm2\.com\/downloads\/.*\.zip)"/); + let m = str.match(/href="(https:\/\/iterm2\.com\/downloads\/.*\.zip)"/); if (m && /iTerm2-[34]/.test(m[1])) { if (m[1]) { links.push(m[1]); @@ -36,10 +45,10 @@ async function getRawReleases() { function transformReleases(links) { let builds = []; for (let link of links) { - var channel = /\/stable\//.test(link) ? 'stable' : 'beta'; + let channel = /\/stable\//.test(link) ? 'stable' : 'beta'; - var parts = link.replace(/.*\/iTerm2[-_]v?(\d_.*)\.zip/, '$1').split('_'); - var version = parts.join('.').replace(/([_-])?beta/, '-beta'); + let parts = link.replace(/.*\/iTerm2[-_]v?(\d_.*)\.zip/, '$1').split('_'); + let version = parts.join('.').replace(/([_-])?beta/, '-beta'); // ex: 3.5.0-beta17 => 3_5_0beta17 // ex: 3.0.2-preview => 3_0_2-preview diff --git a/julia/releases.js b/julia/releases.js index ec0f7f83a..2c00624cf 100644 --- a/julia/releases.js +++ b/julia/releases.js @@ -1,31 +1,61 @@ 'use strict'; -var osMap = { +let Fetcher = require('../_common/fetcher.js'); + +/** @type {Object.} */ +let osMap = { winnt: 'windows', mac: 'darwin', }; -var archMap = { + +/** @type {Object.} */ +let archMap = { armv7l: 'armv7', i686: 'x86', powerpc64le: 'ppc64le', }; +/** + * @typedef BuildInfo + * @prop {String} version + * @prop {String} [_version] + * @prop {String} [arch] + * @prop {String} channel + * @prop {String} date + * @prop {String} download + * @prop {String} [ext] + * @prop {String} [_filename] + * @prop {String} [hash] + * @prop {String} [libc] + * @prop {Boolean} [_musl] + * @prop {Boolean} [lts] + * @prop {String} [size] + * @prop {String} os + */ + async function getDistributables() { let all = { + /** @type {Array} */ releases: [], download: '', _names: ['julia', 'macaarch64'], }; - let resp = await fetch( - 'https://julialang-s3.julialang.org/bin/versions.json', - { - headers: { - Accept: 'application/json', - }, - }, - ); - let buildsByVersion = await resp.json(); + let resp; + try { + let url = 'https://julialang-s3.julialang.org/bin/versions.json'; + resp = await Fetcher.fetch(url, { + headers: { Accept: 'application/json' }, + }); + } catch (e) { + /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error + let err = e; + if (err.code === 'E_FETCH_RELEASES') { + err.message = `failed to fetch 'julia' release data: ${err.response.status} ${err.response.body}`; + } + throw e; + } + let buildsByVersion = JSON.parse(resp.body); /* { @@ -105,6 +135,12 @@ async function getDistributables() { return all; } +/** + * @param {Object} a + * @param {String} a.version + * @param {Object} b + * @param {String} b.version + */ function sortByVersion(a, b) { let [aVer, aPre] = a.version.split('-'); let [bVer, bPre] = b.version.split('-'); diff --git a/macos/releases.js b/macos/releases.js index 3bc9294ad..703f6f066 100644 --- a/macos/releases.js +++ b/macos/releases.js @@ -1,6 +1,8 @@ 'use strict'; -var oses = [ +let Fetcher = require('../_common/fetcher.js'); + +let oses = [ { name: 'macOS Sierra', version: '10.12.6', @@ -25,7 +27,7 @@ var oses = [ }, ]; -var headers = { +let headers = { Connection: 'keep-alive', 'Cache-Control': 'max-age=0', 'Upgrade-Insecure-Requests': '1', @@ -44,18 +46,22 @@ var headers = { * @param {typeof oses[0]} os */ async function fetchReleasesForOS(os) { - let resp = await fetch(os.url, { - headers: headers, - }); - let text = await resp.text(); - if (!resp.ok) { - throw new Error( - `Failed to fetch URL: ${os.url}. HTTP ${resp.status}: ${text}`, - ); + let resp; + try { + resp = await Fetcher.fetch(os.url, { + headers: headers, + }); + } catch (e) { + /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error + let err = e; + if (err.code === 'E_FETCH_RELEASES') { + err.message = `failed to fetch 'macos' release data: ${err.response.status} ${err.response.body}`; + } + throw e; } // Extract the download link - let match = text.match(/(http[^>]+Install[^>]+\.dmg)/); + let match = resp.body.match(/(http[^>]+Install[^>]+\.dmg)/); if (match) { return match[1]; } diff --git a/node/releases.js b/node/releases.js index 3e4fdcc6d..7fc7e917e 100644 --- a/node/releases.js +++ b/node/releases.js @@ -1,8 +1,10 @@ 'use strict'; +let Fetcher = require('../_common/fetcher.js'); + // https://blog.risingstack.com/update-nodejs-8-end-of-life-no-support/ // 6 mos "current" + 18 mos LTS "active" + 12 mos LTS "maintenance" -//var endOfLife = 3 * 366 * 24 * 60 * 60 * 1000; +//let endOfLife = 3 * 366 * 24 * 60 * 60 * 1000; // If there have been no updates in 12 months, it's almost certainly end-of-life const END_OF_LIFE = 366 * 24 * 60 * 60 * 1000; @@ -89,33 +91,43 @@ async function getDistributables() { let baseUrl = `https://nodejs.org/download/release`; // Fetch official builds - let resp = await fetch(`${baseUrl}/index.json`, { - headers: { Accept: 'application/json' }, - }); - let text = await resp.text(); - if (!resp.ok) { - throw new Error( - `Failed to fetch official builds: HTTP ${resp.status}: ${text}`, - ); + let resp; + try { + resp = await Fetcher.fetch(`${baseUrl}/index.json`, { + headers: { Accept: 'application/json' }, + }); + } catch (e) { + /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error + let err = e; + if (err.code === 'E_FETCH_RELEASES') { + err.message = `failed to fetch 'node' release data: ${err.response.status} ${err.response.body}`; + } + throw e; } - let data = JSON.parse(text); + let data = JSON.parse(resp.body); void transform(baseUrl, data); } { - // Fetch unofficial builds let unofficialBaseUrl = `https://unofficial-builds.nodejs.org/download/release`; - let resp = await fetch(`${unofficialBaseUrl}/index.json`, { - headers: { Accept: 'application/json' }, - }); - let text = await resp.text(); - if (!resp.ok) { - throw new Error( - `Failed to fetch official builds: HTTP ${resp.status}: ${text}`, - ); + + // Fetch unofficial builds + let resp; + try { + resp = await Fetcher.fetch(`${unofficialBaseUrl}/index.json`, { + headers: { Accept: 'application/json' }, + }); + } catch (e) { + /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error + let err = e; + if (err.code === 'E_FETCH_RELEASES') { + err.message = `failed to fetch 'node' (unofficial) release data: ${err.response.status} ${err.response.body}`; + } + throw e; } - let data = JSON.parse(text); + let data = JSON.parse(resp.body); + transform(unofficialBaseUrl, data); } diff --git a/terraform/releases.js b/terraform/releases.js index 19c3c1fd7..4a82a69ad 100644 --- a/terraform/releases.js +++ b/terraform/releases.js @@ -1,5 +1,7 @@ 'use strict'; +let Fetcher = require('../_common/fetcher.js'); + /** * @typedef BuildInfo * @prop {String} version @@ -7,16 +9,22 @@ */ async function getDistributables() { - let resp = await fetch( - 'https://releases.hashicorp.com/terraform/index.json', - { headers: { Accept: 'application/json' } }, - ); - let text = await resp.text(); - if (!resp.ok) { - throw new Error(`Failed to fetch releases: HTTP ${resp.status}: ${text}`); + let resp; + try { + let url = 'https://releases.hashicorp.com/terraform/index.json'; + resp = await Fetcher.fetch(url, { + headers: { Accept: 'application/json' }, + }); + } catch (e) { + /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error + let err = e; + if (err.code === 'E_FETCH_RELEASES') { + err.message = `failed to fetch 'terraform' release data: ${err.response.status} ${err.response.body}`; + } + throw e; } + let releases = JSON.parse(resp.body); - let releases = JSON.parse(text); let all = { /** @type {Array} */ releases: [], diff --git a/zig/releases.js b/zig/releases.js index b6fc27c65..1377ff2b6 100644 --- a/zig/releases.js +++ b/zig/releases.js @@ -1,7 +1,9 @@ 'use strict'; -var NON_BUILDS = ['bootstrap', 'src']; -var ODDITIES = NON_BUILDS.concat(['armv6kz-linux']); +let Fetcher = require('../_common/fetcher.js'); + +let NON_BUILDS = ['bootstrap', 'src']; +let ODDITIES = NON_BUILDS.concat(['armv6kz-linux']); /** * @typedef BuildInfo @@ -21,15 +23,21 @@ var ODDITIES = NON_BUILDS.concat(['armv6kz-linux']); */ module.exports = async function () { - let resp = await fetch('https://ziglang.org/download/index.json', { - method: 'GET', - headers: { Accept: 'application/json' }, - }); - let text = await resp.text(); - if (!resp.ok) { - throw new Error(`Failed to fetch releases: HTTP ${resp.status}: ${text}`); + let resp; + try { + let url = 'https://ziglang.org/download/index.json'; + resp = await Fetcher.fetch(url, { + headers: { Accept: 'application/json' }, + }); + } catch (e) { + /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error + let err = e; + if (err.code === 'E_FETCH_RELEASES') { + err.message = `failed to fetch 'zig' release data: ${err.response.status} ${err.response.body}`; + } + throw e; } - let versions = JSON.parse(text); + let versions = JSON.parse(resp.body); /** @type {Array} */ let releases = []; From 6320c519dc6bc6dd65a42e0db9d06ecec8b13e27 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Mon, 16 Dec 2024 00:54:42 +0000 Subject: [PATCH 10/27] fix(terraform): correct channel for stable and non-stable (rc, beta, alpha) --- terraform/releases.js | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/terraform/releases.js b/terraform/releases.js index 4a82a69ad..7c5cd3f88 100644 --- a/terraform/releases.js +++ b/terraform/releases.js @@ -2,6 +2,10 @@ let Fetcher = require('../_common/fetcher.js'); +let alphaRe = /\d-alpha\d/; +let betaRe = /\d-beta\d/; +let rcRe = /\d-rc\d/; + /** * @typedef BuildInfo * @prop {String} version @@ -36,6 +40,18 @@ async function getDistributables() { for (let version of allVersions) { for (let build of releases.versions[version].builds) { + let channel = 'stable'; + let isRc = rcRe.test(version); + let isBeta = betaRe.test(version); + let isAlpha = alphaRe.test(version); + if (isRc) { + channel = 'rc'; + } else if (isBeta) { + channel = 'beta'; + } else if (isAlpha) { + channel = 'alpha'; + } + let r = { version: build.version, download: build.url, @@ -43,7 +59,7 @@ async function getDistributables() { // and the per-file logic has proven to get outdated sooner //os: convert[build.os], //arch: convert[build.arch], - //channel: 'stable|-rc|-beta|-alpha', + channel: channel, }; all.releases.push(r); } From de71f667a0a65412db31f9cc5d545690048b7898 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Sun, 10 Nov 2024 08:25:10 +0000 Subject: [PATCH 11/27] doc(uuidgen): add uppercase and uuidv4 examples --- uuidv7/README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/uuidv7/README.md b/uuidv7/README.md index 71f0bf834..fe66fc319 100644 --- a/uuidv7/README.md +++ b/uuidv7/README.md @@ -53,6 +53,30 @@ uuidv7 ; uuidv7 ; uuidv7 01928d74-3ffb-7e06-abe9-3fe20e5cb5f2 ``` +### How to Generate UPPER CASE (like `uuidgen`) + +```sh +uuidv7 | tr '[:lower:]' '[:upper:]' +``` + +```text +01928D73-D8ED-7211-A314-7081D763271D +``` + +### How to Generate v4 UUIDs? + +Use `uuidgen`. + +```sh +uuidgen +uuidgen | tr '[:upper:]' '[:lower:]' +``` + +```text +84FA79E5-024E-4388-8D10-91618B93BE9D +84fa79e5-024e-4388-8d10-91618b93be9d +``` + ### How could I roll my own UUID v7 generator? It's not that hard. There are examples in many languages here: From 3995b7e5683c8db2f8addf6e10b026fb34844047 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Wed, 11 Dec 2024 23:44:59 +0000 Subject: [PATCH 12/27] feat(serviceman): update for v0.9 --- serviceman/README.md | 203 +++++++++++++++++++++++------------------ serviceman/install.sh | 50 ++++++++-- serviceman/releases.js | 35 +++++-- 3 files changed, 184 insertions(+), 104 deletions(-) diff --git a/serviceman/README.md b/serviceman/README.md index ef5d5eecb..a9011c27d 100644 --- a/serviceman/README.md +++ b/serviceman/README.md @@ -1,6 +1,6 @@ --- title: Serviceman -homepage: https://git.rootprojects.org/root/serviceman +homepage: https://github.com/bnnanet/serviceman tagline: | Serviceman generates and enables startup files on Linux, Mac, and Windows. --- @@ -71,8 +71,7 @@ changes) ### Example: Bash ```sh -sudo env PATH="$PATH" \ - serviceman add --system --path="$PATH" -- \ +serviceman add --name 'backup' -- \ bash ./backup.sh /mnt/data ``` @@ -83,9 +82,7 @@ sudo env PATH="$PATH" \ ```sh pushd ./my-node-app/ -sudo env PATH="$PATH" \ - serviceman add --system --path="$PATH" \ - --cap-net-bind -- \ +serviceman add --name 'my-node-app' -- \ npx nodemon ./server.js ``` @@ -94,9 +91,7 @@ sudo env PATH="$PATH" \ ```sh pushd ./my-node-app/ -sudo env PATH="$PATH" \ - serviceman add --system --path="$PATH" \ - --cap-net-bind -- \ +serviceman add --name 'my-node-app' -- \ npm start ``` @@ -105,9 +100,7 @@ sudo env PATH="$PATH" \ ```sh pushd ./my-go-package/ -sudo env PATH="$PATH" \ - serviceman add --system --path="$PATH" \ - -- \ +serviceman add --name 'my-service' -- \ go run -mod=vendor cmd/my-service/*.go --port 3000 ``` @@ -115,17 +108,15 @@ sudo env PATH="$PATH" \ pushd ./my-go-package/ go build -mod=vendor cmd/my-service -sudo env PATH="$PATH" \ - serviceman add --system --path="$PATH" \ - --cap-net-bind -- \ +serviceman add --name 'my-service' -- \ ./my-service --port 80 ``` ### How to see all services ```sh -serviceman list --system -serviceman list --user +serviceman list --system --all +serviceman list --agent --all ``` ```text @@ -140,8 +131,8 @@ You can either `add` the service again (which will update any changed options), or you can `stop` and then `start` any service by its name: ```sh -sudo env PATH="$PATH" serviceman stop example-service -sudo env PATH="$PATH" serviceman start example-service +serviceman stop 'example-service' +serviceman start 'example-service' ``` ### See the (sub)command help @@ -161,9 +152,7 @@ serviceman add --help ### Use `--dryrun` to see the generated launcher config: ```sh -sudo env PATH="$PATH" \ - serviceman add --system --path="$PATH" \ - --dryrun -- \ +serviceman add --name 'my-backups' --dryrun -- \ bash ./backup.sh /mnt/data ``` @@ -173,26 +162,59 @@ sudo env PATH="$PATH" \ desktop distros. ```text +# Generated for serviceman. Edit as needed. Keep this line for 'serviceman list'. +# https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html + [Unit] -Description=example-service +Description=postgres postgres daemon +Documentation=(none) After=network-online.target Wants=network-online.target systemd-networkd-wait-online.service [Service] Restart=always -StartLimitInterval=10 -StartLimitBurst=3 +RestartSec=3 +RestartSteps=5 +RestartMaxDelaySec=300 -User=root -Group=root +User=app +Group=app -WorkingDirectory=/srv/example-service -ExecStart=/srv/example-service/bin/example-command start +Environment="PATH=/Users/app/.local/opt/pg-essentials/bin:/home/app/.local/opt/postgres/bin:/usr/bin:/bin" +WorkingDirectory=/home/app/.local/share/postgres/var +ExecStart="/home/app/.local/opt/postgres/bin/postgres" "-D" "/home/app/.local/share/postgres/var" "-p" "5432" ExecReload=/bin/kill -USR1 $MAINPID -# Allow the program to bind on privileged ports, such as 80 and 443 -CapabilityBoundingSet=CAP_NET_BIND_SERVICE -AmbientCapabilities=CAP_NET_BIND_SERVICE +# Limit the number of file descriptors and processes; see `man systemd.exec` for more limit settings. +# These are reasonable defaults for a production system. +# Note: systemd "user units" do not support this +LimitNOFILE=1048576 +LimitNPROC=65536 + +# Enable if desired for extra file system security +# (ex: non-containers, multi-user systems) +# +# Use private /tmp and /var/tmp, which are discarded after the service stops. +; PrivateTmp=true +# Use a minimal /dev +; PrivateDevices=true +# Hide /home, /root, and /run/user. Nobody will steal your SSH-keys. +; ProtectHome=true +# Make /usr, /boot, /etc and possibly some more folders read-only. +; ProtectSystem=full +# ... except /opt/{{ .Name }} because we want a place for the database +# and /var/log/{{ .Name }} because we want a place where logs can go. +# This merely retains r/w access rights, it does not add any new. +# Must still be writable on the host! +; ReadWriteDirectories=/opt/postgres /var/log/postgres + +# Grant restricted, root-like privileges to the service. +# CAP_NET_BIND_SERVICE allows binding on privileged ports as a non-root user +# CAP_LEASE allows locking files and is sometimes used for handling file uploads +# Some services may require additional capabilities: +# https://man7.org/linux/man-pages/man7/capabilities.7.html +CapabilityBoundingSet=CAP_NET_BIND_SERVICE CAP_LEASE +AmbientCapabilities=CAP_NET_BIND_SERVICE CAP_LEASE NoNewPrivileges=true [Install] @@ -208,81 +230,88 @@ _container-friendly_ Linuxes. ```sh #!/sbin/openrc-run -supervisor=supervise-daemon - -name="Example System Daemon" -description="A Service for Logging 'Hello, World', a lot!" -description_checkconfig="Check configuration" -description_reload="Reload configuration without downtime" - -# example: -# exampled run --port 1337 --envfile /path/to/env -# exampled check-config --port 1337 --envfile /path/to/env -# exampled reload --port 1337 --envfile /path/to/env -# for setting Config -: ${exampled_opts:="--envfile /root/.config/exampled/env"} +# Generated for serviceman. Edit as needed. Keep this line for 'serviceman list'. +name="postgres" +# docs: (none) +description="postgres daemon" -command=/root/bin/exampled -command_args="run --port 1337 $exampled_opts" -command_user=root:root -extra_commands="checkconfig" -extra_started_commands="reload" -output_log=/var/log/exampled.log -error_log=/var/log/exampled.err +supervisor="supervise-daemon" +output_log="/var/log/postgres" +error_log="/var/log/postgres" depend() { - need net localmount - after firewall + need net } -checkconfig() { - ebegin "Checking configuration for $name" - su ${command_user%:*} -s /bin/sh -c "$command check-config $exampled_opts" - eend $? +start_pre() { + checkpath --directory --owner root /var/log/ + checkpath --file --owner 'app:app' ${output_log} ${error_log} } -reload() { - ebegin "Reloading $name" - su ${command_user%:*} -s /bin/sh -c "$command reload $exampled_opts" +start() { + ebegin "Starting ${name}" + supervise-daemon ${name} --start \ + --chdir '/home/app/.local/share/postgres/var' \ + --env 'PATH=/Users/app/.local/opt/pg-essentials/bin:/home/app/.local/opt/postgres/bin:/usr/bin:/bin' \ + --user 'app' \ + --group 'app' \ + --stdout ${output_log} \ + --stderr ${error_log} \ + --pidfile /run/${RC_SVCNAME}.pid \ + --respawn-delay 5 \ + --respawn-max 51840 \ + --capabilities=CAP_NET_BIND_SERVICE \ + -- \ + '/home/app/.local/opt/postgres/bin/postgres' '-D' '/home/app/.local/share/postgres/var' '-p' '5432' eend $? } -stop_pre() { - if [ "$RC_CMD" = restart ]; then - checkconfig || return $? - fi +stop() { + ebegin "Stopping ${name}" + supervise-daemon ${name} --stop \ + --pidfile /run/${RC_SVCNAME}.pid + eend $? } ``` ### What a typical launchd .plist file looks like -```text +```xml - + - Label - example-service - ProgramArguments - - /Users/me/example-service/bin/example-command - start - - - RunAtLoad - - KeepAlive - - - WorkingDirectory - /Users/me/example-service - - StandardErrorPath - /Users/me/.local/share/example-service/var/log/example-service.log - StandardOutPath - /Users/me/.local/share/example-service/var/log/example-service.log + Label + postgres + ProgramArguments + + /Users/app/.local/opt/postgres/bin/postgres + -D + /Users/app/.local/share/postgres/var + -p + 5432 + + + EnvironmentVariables + + PATH + /Users/app/.local/opt/pg-essentials/bin:/Users/app/.local/opt/postgres/bin:/usr/bin:/bin + + + RunAtLoad + + KeepAlive + + + WorkingDirectory + /Users/app/.local/share/postgres/var + + StandardOutPath + /Users/app/.local/share/postgres/var/log/postgres.log + StandardErrorPath + /Users/app/.local/share/postgres/var/log/postgres.log ``` diff --git a/serviceman/install.sh b/serviceman/install.sh index 7422b4882..c7844a875 100644 --- a/serviceman/install.sh +++ b/serviceman/install.sh @@ -12,28 +12,62 @@ __init_serviceman() { pkg_cmd_name="serviceman" pkg_dst_cmd="$HOME/.local/bin/serviceman" + # shellcheck disable=SC2034 pkg_dst="$pkg_dst_cmd" pkg_src_cmd="$HOME/.local/opt/serviceman-v$WEBI_VERSION/bin/serviceman" + pkg_src_bin="$HOME/.local/opt/serviceman-v$WEBI_VERSION/bin" pkg_src_dir="$HOME/.local/opt/serviceman-v$WEBI_VERSION" + # shellcheck disable=SC2034 pkg_src="$pkg_src_cmd" pkg_install() { - # $HOME/.local/opt/serviceman-v0.8.0/bin - mkdir -p "$pkg_src_bin" + if test -e ./*"$pkg_cmd_name"*/share; then + rm -rf "${pkg_src_dir}" + # mv ./bnnanet-serviceman-* "$HOME/.local/opt/serviceman-v0.9.1" + mv ./*"$pkg_cmd_name"*/ "${pkg_src_dir}" + else + echo "NO share" + # $HOME/.local/opt/serviceman-v0.8.0/bin + mkdir -p "$pkg_src_bin" - # mv ./serviceman* "$HOME/.local/opt/serviceman-v0.8.0/bin/serviceman" - mv ./"$pkg_cmd_name"* "$pkg_src_cmd" + # mv ./serviceman* "$HOME/.local/opt/serviceman-v0.8.0/bin/serviceman" + mv ./"$pkg_cmd_name"* "$pkg_src_cmd" - # chmod a+x "$HOME/.local/opt/serviceman-v0.8.0/bin/serviceman" - chmod a+x "$pkg_src_cmd" + # chmod a+x "$HOME/.local/opt/serviceman-v0.8.0/bin/serviceman" + chmod a+x "$pkg_src_cmd" + fi + } + + pkg_link() { + ( + cd ~/.local/opt/ || return 1 + rm -rf ./serviceman + ln -s "serviceman-v$WEBI_VERSION" 'serviceman' + ) + + ( + mkdir -p ~/.local/share/ + cd ~/.local/share/ || return 1 + rm -rf ./serviceman + ln -s "../opt/serviceman-v$WEBI_VERSION/share/serviceman" 'serviceman' + ) + + ( + mkdir -p ~/.local/bin/ + cd ~/.local/bin/ || return 1 + rm -rf ./serviceman + ln -s "../opt/serviceman-v$WEBI_VERSION/bin/serviceman" 'serviceman' + ) } pkg_get_current_version() { # 'serviceman version' has output in this format: - # serviceman v0.8.0 (f3ab547) 2020-12-02T16:19:10-07:00 + # serviceman v0.9.1 (2024-12-11 14:29 -0500) + # Copyright 2024 AJ ONeal + # Licensed under the MPL-2.0 # This trims it down to just the version number: - # 0.8.0 + # 0.9.1 serviceman --version 2> /dev/null | head -n 1 | cut -d' ' -f2 | sed 's:^v::' } diff --git a/serviceman/releases.js b/serviceman/releases.js index 2c7634aa8..91e3274d2 100644 --- a/serviceman/releases.js +++ b/serviceman/releases.js @@ -1,18 +1,35 @@ 'use strict'; -var github = require('../_common/github.js'); -var owner = 'therootcompany'; -var repo = 'serviceman'; +let Releases = module.exports; -module.exports = function () { - return github(null, owner, repo).then(function (all) { - return all; - }); +let GitHub = require('../_common/github.js'); +let oldOwner = 'therootcompany'; +let oldRepo = 'serviceman'; + +let GitHubSource = require('../_common/github-source.js'); +let owner = 'bnnanet'; +let repo = 'serviceman'; + +Releases.latest = async function () { + let all = await GitHubSource.getDistributables({ owner, repo }); + for (let pkg of all.releases) { + //@ts-expect-error + pkg.os = 'posix_2017'; + } + + let all2 = await GitHub.getDistributables(null, oldOwner, oldRepo); + for (let pkg of all2.releases) { + //@ts-expect-error + all.releases.push(pkg); + } + + return all; }; if (module === require.main) { - module.exports().then(function (all) { + //@ts-expect-error + Releases.latest().then(function (all) { all = require('../_webi/normalize.js')(all); - console.info(JSON.stringify(all)); + console.info(JSON.stringify(all, null, 2)); }); } From e2ad1970678e92c28ea879a1a1700c5af596372c Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Thu, 12 Dec 2024 00:05:55 +0000 Subject: [PATCH 13/27] doc(serviceman): --agent instead of --user --- brew/brew-update-service-install | 2 +- bun/README.md | 2 +- caddy/README.md | 6 +++--- node/README.md | 5 +++-- syncthing/README.md | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/brew/brew-update-service-install b/brew/brew-update-service-install index 5f26baad8..0e72d16d9 100644 --- a/brew/brew-update-service-install +++ b/brew/brew-update-service-install @@ -13,7 +13,7 @@ main() { ( serviceman --version fi - env PATH="$PATH" serviceman add --user \ + env PATH="$PATH" serviceman add --agent \ --workdir ~/.local/opt/brew/ \ --name sh.brew.updater -- \ ~/.local/bin/brew-update-hourly diff --git a/bun/README.md b/bun/README.md index dd7c74240..4a7581e57 100644 --- a/bun/README.md +++ b/bun/README.md @@ -155,6 +155,6 @@ For **macOS**: ``` 3. Add your project to the system launcher, running as the current user ```sh - serviceman add --path="$PATH" --user --name my-project -- \ + serviceman add --path="$PATH" --agent --name my-project -- \ bun run ./my-project.js ``` diff --git a/caddy/README.md b/caddy/README.md index 4d812cff2..76e6ca35f 100644 --- a/caddy/README.md +++ b/caddy/README.md @@ -821,7 +821,7 @@ To avoid the nitty-gritty details of `launchd` plist files, you can use ```sh my_username="$(id -u -n)" - serviceman add --user --name caddy -- \ + serviceman add --agent --name caddy -- \ caddy run --config ./Caddyfile --envfile ~/.config/caddy/env ``` @@ -915,9 +915,9 @@ See the notes below to run as a **User Service** or use the JSON Config. To create a **User Service** instead: -- don't use `sudo`, but do use `--user` when running `serviceman`: +- don't use `sudo`, but do use `--agent` when running `serviceman`: ```sh - serviceman add --user --name caddy -- \ + serviceman add --agent --name caddy -- \ caddy run --config ./Caddyfile --envfile ~/.config/caddy/env ``` (this will create `~/.config/systemd/user/`) diff --git a/node/README.md b/node/README.md index 59034c1f8..23841f9ab 100644 --- a/node/README.md +++ b/node/README.md @@ -66,7 +66,8 @@ To run them manually on your code; jhint -c ./.jshintrc *.js */*.js ``` - fixjson \ - (turns JavaScript Objects with comments, trailing commas, etc into actual json) + (turns JavaScript Objects with comments, trailing commas, etc into actual + json) ```sh fixjson -i 2 -w ./package.json ``` @@ -229,7 +230,7 @@ Node app as a Non-System (Unprivileged) Service on Mac, Windows, and Linux: ```sh my_username="$(id -u -n)" - serviceman add --user --name my-node-project -- \ + serviceman add --agent --name my-node-project -- \ caddy run --config ./Caddyfile --envfile ~/.config/caddy/env ``` diff --git a/syncthing/README.md b/syncthing/README.md index b0b5885ba..5b7fbbd00 100644 --- a/syncthing/README.md +++ b/syncthing/README.md @@ -46,7 +46,7 @@ webi serviceman ```sh mkdir -p ~/.config/syncthing/ -env PATH="$PATH" serviceman add --user --name syncthing -- \ +env PATH="$PATH" serviceman add --agent --name syncthing -- \ syncthing --home ~/.config/syncthing/ ``` From 40316a866c3cf3ff2140594c363e3193525bef7c Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Thu, 12 Dec 2024 00:45:11 +0000 Subject: [PATCH 14/27] doc(serviceman): update docs across installers --- brew/brew-update-service-install | 4 +-- bun/README.md | 6 ++-- caddy/README.md | 37 ++++++++++--------------- dashcore-utils/README.md | 3 +- dashcore-utils/dashd-hd-service-install | 34 ++++++++--------------- dashd/README.md | 15 ++-------- go/README.md | 3 +- golang/README.md | 3 +- node/README.md | 13 +++------ pg/install.sh | 9 ++++-- postgres/README.md | 8 ++---- postgres/install.sh | 9 ++++-- 12 files changed, 54 insertions(+), 90 deletions(-) diff --git a/brew/brew-update-service-install b/brew/brew-update-service-install index 0e72d16d9..262b95c0c 100644 --- a/brew/brew-update-service-install +++ b/brew/brew-update-service-install @@ -7,11 +7,11 @@ main() { ( chmod a+x ~/.local/bin/brew-update-hourly echo "Checking for serviceman..." + ~/.local/bin/webi serviceman if ! command -v serviceman > /dev/null; then - "$HOME/.local/bin/webi" serviceman export PATH="$HOME/.local/bin:$PATH" - serviceman --version fi + serviceman --version env PATH="$PATH" serviceman add --agent \ --workdir ~/.local/opt/brew/ \ diff --git a/bun/README.md b/bun/README.md index 4a7581e57..e688ebd60 100644 --- a/bun/README.md +++ b/bun/README.md @@ -132,9 +132,7 @@ file) ``` 3. Add your project to the system launcher, running as the current user ```sh - sudo env PATH="$PATH" \ - serviceman add --path="$PATH" --system \ - --username "$(id -u -n)" --name my-project -- \ + serviceman add --name 'my-project' --daemon -- \ bun run ./my-project.js ``` 4. Restart the logging service @@ -155,6 +153,6 @@ For **macOS**: ``` 3. Add your project to the system launcher, running as the current user ```sh - serviceman add --path="$PATH" --agent --name my-project -- \ + serviceman add --agent --name 'my-project' -- \ bun run ./my-project.js ``` diff --git a/caddy/README.md b/caddy/README.md index 76e6ca35f..c6264fd68 100644 --- a/caddy/README.md +++ b/caddy/README.md @@ -821,8 +821,8 @@ To avoid the nitty-gritty details of `launchd` plist files, you can use ```sh my_username="$(id -u -n)" - serviceman add --agent --name caddy -- \ - caddy run --config ./Caddyfile --envfile ~/.config/caddy/env + serviceman add --agent --name 'caddy' --workdir ./ -- \ + caddy run --envfile ~/.config/caddy/env --config ./Caddyfile --adapter caddyfile ``` (this will create `~/Library/LaunchAgents/caddy.plist`) @@ -837,8 +837,8 @@ This process creates a _User-Level_ service in `~/Library/LaunchAgents`. To create a _System-Level_ service in `/Library/LaunchDaemons/` instead: ```sh -sudo serviceman add --system --name caddy -- \ - caddy run --config ./Caddyfile --envfile ~/.config/caddy/env +serviceman add --name 'caddy' --workdir ./ --daemon -- \ + caddy run --envfile ~/.config/caddy/env --config ./Caddyfile --adapter caddyfile ``` ### How to run Caddy as a Windows Service @@ -856,7 +856,7 @@ sudo serviceman add --system --name caddy -- \ 3. Create a **Startup Registry Entry** with Serviceman. ```sh serviceman.exe add --name caddy -- \ - caddy run --config ./Caddyfile --envfile ~/.config/caddy/env + caddy run --envfile ~/.config/caddy/env --config ./Caddyfile --adapter caddyfile ``` 4. You can manage the service directly with Serviceman. For example: ```sh @@ -901,10 +901,8 @@ See the notes below to run as a **User Service** or use the JSON Config. ``` 4. Use Serviceman to create a _systemd_ config file. ```sh - my_username="$(id -u -n)" - sudo env PATH="$PATH" \ - serviceman add --system --username "${my_username}" --name caddy -- \ - caddy run --config ./Caddyfile --envfile ~/.config/caddy/env + serviceman add --name 'caddy' --daemon -- \ + caddy run --envfile ~/.config/caddy/env --config ./Caddyfile --adapter caddyfile ``` (this will create `/etc/systemd/system/caddy.service`) 5. Manage the service with `systemctl` and `journalctl`: @@ -915,10 +913,10 @@ See the notes below to run as a **User Service** or use the JSON Config. To create a **User Service** instead: -- don't use `sudo`, but do use `--agent` when running `serviceman`: +- use `--agent` when running `serviceman`: ```sh serviceman add --agent --name caddy -- \ - caddy run --config ./Caddyfile --envfile ~/.config/caddy/env + caddy run --envfile ~/.config/caddy/env --config ./Caddyfile --adapter caddyfile ``` (this will create `~/.config/systemd/user/`) - user the `--user` flag to manage services and logs: @@ -1183,7 +1181,8 @@ To prevent search engine and browser confusion - _DO NOT_ prevent crawling via `robots.txt` \ (counter-intuitive, but pages _must_ be crawled for links to _NOT_ be indexed) - _all_ domains using public TLS certs _will_ be indexed by default \ - (they are all linked to and crawled from various Certificate Transparency reports) + (they are all linked to and crawled from various Certificate Transparency + reports) - follow these guidelines even if the dev sites use HTTP Basic Auth ```Caddyfile @@ -1363,19 +1362,13 @@ See also: 2. Generate the `service` file: \ - JSON Config ```sh - my_app_user="$(id -u -n)" - sudo env PATH="${PATH}" \ - serviceman add --system --cap-net-bind \ - --username "${my_app_user}" --name caddy -- \ - caddy run --resume --envfile ./caddy.env + serviceman add --name 'caddy' --daemon -- \ + caddy run --resume --envfile ./caddy.env ``` - Caddyfile ```sh - my_app_user="$(id -u -n)" - sudo env PATH="${PATH}" \ - serviceman add --system --cap-net-bind \ - --username "${my_app_user}" --name caddy -- \ - caddy run --config ./Caddyfile --envfile ./caddy.env + serviceman add --name 'caddy' --daemon -- \ + caddy run --config ./Caddyfile --envfile ./caddy.env ``` 3. Reload `systemd` config files, the logging service (it may not be started on a new VPS), and caddy diff --git a/dashcore-utils/README.md b/dashcore-utils/README.md index 41c54edcc..f2e3d4d08 100644 --- a/dashcore-utils/README.md +++ b/dashcore-utils/README.md @@ -100,8 +100,7 @@ mkdir -p ~/.dashcore/wallets/ mkdir -p /mnt/slc1_vol_100g/dashcore/_data mkdir -p /mnt/slc1_vol_100g/dashcore/_caches -sudo env PATH="$PATH" serviceman add \ - --system --user "$my_user" --path "$PATH" --name dashd --force -- \ +serviceman add --name 'dashd' --daemon -- \ dashd \ -usehd \ -conf="$HOME/.dashcore/dash.conf" \ diff --git a/dashcore-utils/dashd-hd-service-install b/dashcore-utils/dashd-hd-service-install index 13aa9b299..ce9b65e92 100644 --- a/dashcore-utils/dashd-hd-service-install +++ b/dashcore-utils/dashd-hd-service-install @@ -84,20 +84,8 @@ fn_srv_install() { ( my_name="dashd-${my_netname}" fi - my_system_args="" - my_kernel="$( - uname -s - )" - if test "Darwin" != "${my_kernel}"; then - my_user="$( - id -u -n - )" - my_system_args="--system --username ${my_user}" - fi - # shellcheck disable=SC2016,SC1090 - echo 'sudo env PATH="$PATH"' \ - "serviceman add ${my_system_args} --path \"\$PATH\" --name \"${my_name}\" --force --" \ + echo "serviceman add --name \"${my_name}\" --" \ "dashd " \ "${my_net_flag}" \ -usehd \ @@ -107,16 +95,16 @@ fn_srv_install() { ( "-datadir=\"${my_datadir}\"" \ "-blocksdir=\"${my_blocksdir}\"" + echo "" + echo "Installing latest 'serviceman'..." + echo "" + "$HOME/.local/bin/webi" serviceman > /dev/null if ! command -v serviceman > /dev/null; then - echo "" - echo "Installing 'serviceman'..." - echo "" - { - "$HOME/.local/bin/webi" serviceman - } > /dev/null - - # shellcheck disable=SC1090 - . ~/.config/envman/PATH.env || true + export PATH="$HOME/.local/bin:$PATH" + fi + serviceman --version + if ! command -v dashd > /dev/null; then + export PATH="$HOME/.local/opt/dashcore/bin:$PATH" fi mkdir -p "$HOME/.dashcore/wallets/" @@ -132,7 +120,7 @@ fn_srv_install() { ( # leave options unquoted so they're interpreted separately # shellcheck disable=SC2086 sudo env PATH="${PATH}" \ - serviceman add ${my_system_args} --path "${PATH}" --name "${my_name}" --force -- \ + serviceman add --name "${my_name}" -- \ dashd \ ${my_net_flag} \ -usehd \ diff --git a/dashd/README.md b/dashd/README.md index 2dd32d413..a89ce5675 100644 --- a/dashd/README.md +++ b/dashd/README.md @@ -219,14 +219,7 @@ You can use [`serviceman`](../serviceman/): **Linux** ```sh -sudo env PATH="$PATH" \ - serviceman add \ - --system \ - --username "$(id -n -u)" \ - --path "$PATH" \ - --name dashd \ - --force \ - -- \ +serviceman add --name 'dashd' -- \ dashd \ -usehd \ -conf="$HOME/.dashcore/dash.conf" \ @@ -239,11 +232,7 @@ sudo env PATH="$PATH" \ **Mac** ```sh -serviceman add \ - --path "$PATH" \ - --name dashd \ - --force \ - -- \ +serviceman add --name 'dashd' -- \ dashd \ -usehd \ -conf="$HOME/.dashcore/dash.conf" \ diff --git a/go/README.md b/go/README.md index 68b415a79..f1b074a67 100644 --- a/go/README.md +++ b/go/README.md @@ -80,8 +80,7 @@ webi serviceman pushd ./hello/ # swap 'hello' and './hello' for the name of your project and binary -sudo env PATH="$PATH" \ - serviceman add --system --username "$(id -u -n)" --name hello -- \ +serviceman add --name 'hello' -- \ ./hello # Restart the logging service diff --git a/golang/README.md b/golang/README.md index 8c064775c..934d75c72 100644 --- a/golang/README.md +++ b/golang/README.md @@ -85,8 +85,7 @@ webi serviceman pushd ./hello/ # swap 'hello' and './hello' for the name of your project and binary -sudo env PATH="$PATH" \ - serviceman add --system --username "$(id -u -n)" --name hello -- \ +serviceman add --name 'hello' -- \ ./hello # Restart the logging service diff --git a/node/README.md b/node/README.md index 23841f9ab..0b6c23a05 100644 --- a/node/README.md +++ b/node/README.md @@ -276,11 +276,8 @@ Node app as a Non-System (Unprivileged) Service on Mac, Windows, and Linux: ```sh pushd ./my-node-project/ -my_username="$(id -u -n)" -sudo env PATH="$PATH" \ - serviceman add --system --path "$PATH" --cap-net-bind \ - --name my-node-project --username "${my_username}" -- \ - npm run start +serviceman add --name 'my-node-project' -- \ + npm run start ``` #### ... with auto-reload in Dev @@ -288,10 +285,8 @@ sudo env PATH="$PATH" \ ```sh pushd ./my-node-project/ -sudo env PATH="$PATH" \ - serviceman add --system --path "$PATH" --cap-net-bind \ - --name my-node-project --username "$(id -u -n)" -- \ - npx -p nodemon@3 -- nodemon ./server.js +serviceman add --name 'my-node-project' -- \ + npx -p nodemon@3 -- nodemon ./server.js ``` #### View Logs & Restart diff --git a/pg/install.sh b/pg/install.sh index 5c388190a..8895a2550 100644 --- a/pg/install.sh +++ b/pg/install.sh @@ -58,17 +58,20 @@ __init_pg() { } pkg_done_message() { - # TODO show with serviceman echo " Installed $(t_pkg "$pkg_cmd_name v$WEBI_VERSION") (and $(t_pkg "psql")) to $(t_link "$(fn_sub_home "${pkg_dst_bin}")")" echo "" echo "IMPORTANT!!!" echo "" - echo "Database initialized at $POSTGRES_DATA_DIR:" - echo " postgres -D $POSTGRES_DATA_DIR -p 5432" + echo "Database initialized at:" + echo " $POSTGRES_DATA_DIR" echo "" echo "Username and password set to 'postgres':" echo " psql 'postgres://postgres:postgres@localhost:5432/postgres'" echo "" + echo "To install as a service:" + echo " serviceman add --name 'postgres' --workdir '$POSTGRES_DATA_DIR' -- \\" + echo " postgres -D '$POSTGRES_DATA_DIR' -p 5432" + echo "" } } diff --git a/postgres/README.md b/postgres/README.md index 6f1e89b25..436f8be2b 100644 --- a/postgres/README.md +++ b/postgres/README.md @@ -34,8 +34,7 @@ To enable Postgres as a Linux Service with [serviceman](../serviceman/): \ (see macOS below) ```sh -sudo env PATH="$PATH" \ - serviceman add --system --username "$(id -u -n)" --name 'postgres' -- \ +serviceman add --name 'postgres' --workdir ~/.local/share/postgres/var -- \ postgres -D ~/.local/share/postgres/var -p 5432 sudo systemctl restart systemd-journald @@ -119,8 +118,7 @@ curl https://webi.sh/serviceman | sh ``` ```sh -sudo env PATH="$PATH" \ - serviceman add --system --username "$(id -u -n)" --name 'postgres' -- \ +serviceman add --name 'postgres' --workdir ~/.local/share/postgres/var -- \ postgres -D ~/.local/share/postgres/var -p 5432 sudo systemctl restart systemd-journald @@ -185,7 +183,7 @@ sudo tail -f /var/log/postgres #### macOS ```sh -serviceman add --name 'postgres' -- \ +serviceman add --name 'postgres' --workdir ~/.local/share/postgres/var -- \ postgres -D ~/.local/share/postgres/var -p 5432 tail -f ~/.local/share/postgres/var/log/postgres.log diff --git a/postgres/install.sh b/postgres/install.sh index 6a0d46fcf..eccf1f6ee 100644 --- a/postgres/install.sh +++ b/postgres/install.sh @@ -78,17 +78,20 @@ __init_postgres() { } pkg_done_message() { - # TODO show with serviceman echo "Installed 'postgres' and 'psql' at $pkg_dst" echo "" echo "IMPORTANT!!!" echo "" - echo "Database initialized at $POSTGRES_DATA_DIR:" - echo " postgres -D $POSTGRES_DATA_DIR -p 5432" + echo "Database initialized at:" + echo " $POSTGRES_DATA_DIR" echo "" echo "Username and password set to 'postgres':" echo " psql 'postgres://postgres:postgres@localhost:5432/postgres'" echo "" + echo "To install as a service:" + echo " serviceman add --name 'postgres' --workdir '$POSTGRES_DATA_DIR' -- \\" + echo " postgres -D '$POSTGRES_DATA_DIR' -p 5432" + echo "" } } From 83a6d02d50fda8413de0b6f4f5d19d9404319b4b Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Sun, 15 Dec 2024 05:27:00 +0000 Subject: [PATCH 15/27] fix(api): project 'alias'es (symlinks) should be resolved before checking for 'selfhosted' --- _webi/serve-installer.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/_webi/serve-installer.js b/_webi/serve-installer.js index 21939993c..02d1f8aa8 100644 --- a/_webi/serve-installer.js +++ b/_webi/serve-installer.js @@ -70,18 +70,20 @@ InstallerServer.helper = async function ({ console.log(`dbg: Get Project Installer Type for '${projectName}':`); let proj = await Builds.getProjectType(projectName); - console.log(proj); + if (proj.type === 'alias') { + console.log(`dbg: alias`, proj); + projectName = proj.detail; + proj = await Builds.getProjectType(projectName); // an alias should never resolve to an alias + } + console.log(`dbg: proj`, proj); - let validTypes = ['alias', 'selfhosted', 'valid']; + let validTypes = ['selfhosted', 'valid']; if (!validTypes.includes(proj.type)) { let msg = `'${projectName}' doesn't have an installer: '${proj.type}': '${proj.detail}'`; let err = new Error(msg); err.code = 'ENOENT'; throw err; } - if (proj.type === 'alias') { - projectName = proj.detail; - } let tmplParams = { pkg: projectName, From 6aeb60008b3eb0146bed64105b5c4a271efa87b7 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Sun, 1 Dec 2024 07:03:03 +0000 Subject: [PATCH 16/27] fix(bun): mark musl builds as hard-musl (not gnu-compatible) --- bun/releases.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bun/releases.js b/bun/releases.js index 1238c15f3..b54035d6f 100644 --- a/bun/releases.js +++ b/bun/releases.js @@ -18,6 +18,14 @@ module.exports = function () { return false; } + let isMusl = r.name.includes('-musl'); + if (isMusl) { + r._musl = true; + r.libc = 'musl'; + } else if (r.os === 'linux') { + r.libc = 'gnu'; + } + return true; }) .map(function (r) { From ce18bd5e611ee1393c28579567d325881b635f3b Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Wed, 11 Dec 2024 22:05:40 +0000 Subject: [PATCH 17/27] doc(gh-source): make baseurl optional --- _common/github-source.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_common/github-source.js b/_common/github-source.js index 42549b161..b6f4e6d98 100644 --- a/_common/github-source.js +++ b/_common/github-source.js @@ -10,7 +10,7 @@ let GitHubishSource = require('./githubish-source.js'); * @param {Object} opts * @param {String} opts.owner * @param {String} opts.repo - * @param {String} opts.baseurl + * @param {String} [opts.baseurl] * @param {String} [opts.username] * @param {String} [opts.token] */ From 117ee6117d2d0d0215206641c988c8ef92803f89 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Wed, 11 Dec 2024 22:07:54 +0000 Subject: [PATCH 18/27] doc(node): add types to package --- package-lock.json | 24 ++++++++++++++++++++---- package.json | 3 +++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2b6fbe908..1e54e99af 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,12 +11,23 @@ "dependencies": { "dotenv": "^16.4.7", "marked": "^15.0.4" + }, + "devDependencies": { + "@types/node": "^22.10.2" + } + }, + "node_modules/@types/node": { + "version": "22.10.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.2.tgz", + "integrity": "sha512-Xxr6BBRCAOQixvonOye19wnzyDiUtTeqldOOmj3CkeblonbccA12PFwlufvRdrpjXxqnmUaeiU5EOA+7s5diUQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.20.0" } }, "node_modules/dotenv": { "version": "16.4.7", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz", - "integrity": "sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ==", "license": "BSD-2-Clause", "engines": { "node": ">=12" @@ -27,8 +38,6 @@ }, "node_modules/marked": { "version": "15.0.4", - "resolved": "https://registry.npmjs.org/marked/-/marked-15.0.4.tgz", - "integrity": "sha512-TCHvDqmb3ZJ4PWG7VEGVgtefA5/euFmsIhxtD0XsBxI39gUSKL81mIRFdt0AiNQozUahd4ke98ZdirExd/vSEw==", "license": "MIT", "bin": { "marked": "bin/marked.js" @@ -36,6 +45,13 @@ "engines": { "node": ">= 18" } + }, + "node_modules/undici-types": { + "version": "6.20.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", + "dev": true, + "license": "MIT" } } } diff --git a/package.json b/package.json index 796061cda..afa06cdf4 100644 --- a/package.json +++ b/package.json @@ -40,5 +40,8 @@ "dependencies": { "dotenv": "^16.4.7", "marked": "^15.0.4" + }, + "devDependencies": { + "@types/node": "^22.10.2" } } From 7b8f882d80d82e8e6a61457a5cb62799bc3dd413 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Mon, 16 Dec 2024 01:47:39 +0000 Subject: [PATCH 19/27] fix(serviceman): do not use 'sudo' or 'env PATH="$PATH"' --- brew/brew-update-service-install | 2 +- dashcore-utils/dashd-hd-service-install | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/brew/brew-update-service-install b/brew/brew-update-service-install index 262b95c0c..a97b677b0 100644 --- a/brew/brew-update-service-install +++ b/brew/brew-update-service-install @@ -13,7 +13,7 @@ main() { ( fi serviceman --version - env PATH="$PATH" serviceman add --agent \ + serviceman add --agent \ --workdir ~/.local/opt/brew/ \ --name sh.brew.updater -- \ ~/.local/bin/brew-update-hourly diff --git a/dashcore-utils/dashd-hd-service-install b/dashcore-utils/dashd-hd-service-install index ce9b65e92..a25a7f854 100644 --- a/dashcore-utils/dashd-hd-service-install +++ b/dashcore-utils/dashd-hd-service-install @@ -119,8 +119,7 @@ fn_srv_install() { ( cd "${my_vol}" || return 1 # leave options unquoted so they're interpreted separately # shellcheck disable=SC2086 - sudo env PATH="${PATH}" \ - serviceman add --name "${my_name}" -- \ + serviceman add --name "${my_name}" -- \ dashd \ ${my_net_flag} \ -usehd \ From 4eff5b6cbec942b8c3aa1ed42ae547dad24813d9 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Mon, 16 Dec 2024 01:48:26 +0000 Subject: [PATCH 20/27] doc(syncthing): 'env PATH=' is no longer needed for serviceman --- syncthing/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncthing/README.md b/syncthing/README.md index 5b7fbbd00..43f4f86e9 100644 --- a/syncthing/README.md +++ b/syncthing/README.md @@ -46,7 +46,7 @@ webi serviceman ```sh mkdir -p ~/.config/syncthing/ -env PATH="$PATH" serviceman add --agent --name syncthing -- \ +serviceman add --agent --name syncthing -- \ syncthing --home ~/.config/syncthing/ ``` From d3f3ad1688c6860b196c189186aa40f48ee3bee5 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Tue, 17 Dec 2024 19:56:46 +0000 Subject: [PATCH 21/27] doc(shellcheck): include ignore/enable code list, update usage and doc links --- shellcheck/README.md | 108 +++++++++++++++++++++++++++++++++++++++--- shellcheck/install.sh | 4 ++ 2 files changed, 105 insertions(+), 7 deletions(-) diff --git a/shellcheck/README.md b/shellcheck/README.md index 50c53a401..af8d3cbb8 100644 --- a/shellcheck/README.md +++ b/shellcheck/README.md @@ -14,6 +14,7 @@ These are the files / directories that are created and/or modified with this install: ```text +~/.shellcheckrc ~/.config/envman/PATH.env ~/.local/opt/shellcheck/ ~/.local/bin/shellcheck @@ -32,6 +33,16 @@ Also recommended by Google's shellcheck ./script.sh ``` +With common options: + +```sh +shellcheck \ + -s sh -S style \ + -e SC1090 -e SC1091 \ + -o add-default-case -o deprecate-which \ + scripts/* +``` + ### How to run shellcheck in vim `shellcheck` is @@ -56,20 +67,103 @@ check-scripts: shellcheck myscripts/*.sh ``` -### How to ignore an error +### How to Enable or Ignore Checks + +You can use SCXXXX codes to disable shellcheck errors and warnings at any level +through a variety of means, also described at +, +, and `shellcheck --list-optional`. -You can ignore an error by putting a comment with the `SCXXXX` error code above -it: +**Single Execution** ```sh -# shellcheck disable= +shellcheck -s sh -S style --exclude=SC1090,SC1091 --enable=add-default-case */*.sh ``` +**Single Line** + +(place directly above the offending line) + ```sh -# shellcheck disable=SC1004 -NOT_AN_ERROR='Look, a literal \ -inside of a string!' +# shellcheck disable=SC2016,SC2088 enable=require-variable-braces +echo '~/ is an alias for $HOME' +``` + +**Whole Function** + +(place directly above the function definition) + +```sh +# shellcheck disable=SC2016,SC2088 enable=require-variable-braces +fn_help() { ( + echo '~/ is an alias for $HOME' +); } +``` + +**Whole File** + +(place directly under the shebang, before any expressions) + +```sh +#!/bin/sh +# shellcheck disable=SC1090,SC1091 enable=require-variable-braces +``` + +**Global Process** + +```sh +export SHELLCHECK_OPTS="-e SC1090 -e SC1091 -o deprecate-which" +``` + +**Global Config** + +`~/.shellcheckrc`: + +```sh +disable=SC1090,SC1091 +disable=SC2155 +enable=add-default-case,check-extra-masked-returns,deprecate-which +enable=quote-safe-variables,check-set-e-suppressed,require-variable-braces +``` + +### Common Ignored & Optional Shellcheck Codes + +```text +SC1003 - Want to escape a single quote? echo 'This is how it'\''s done'. +SC1004 - This backslash+linefeed is literal. Break outside single quotes if you just want to break the line. +SC1090 - Can't follow non-constant source. Use a directive to specify location. +SC1091 - Not following: (error message here) # for source .env, etc +SC2005 - Useless `echo`? Instead of `echo $(cmd)`, just use `cmd` +SC2010 - Don't use ls | grep. Use a glob or a for loop with a condition to allow non-alphanumeric filenames. +SC2016 - Expressions don't expand in single quotes, use double quotes for that. +SC2029 - Note that, unescaped, this expands on the client side. +SC2046 - Quote this to prevent word splitting +SC2059 - Don't use variables in the printf format string. Use printf "..%s.." "$foo". +SC2072 - Decimals are not supported. Either use integers only, or use bc or awk to compare. +SC2086 - Double quote to prevent globbing and word splitting. +SC2087 - Quote 'EOF' to make here document expansions happen on the server side rather than on the client. +SC2088 - Tilde does not expand in quotes. Use $HOME. +SC2155 - Declare and assign separately to avoid masking return values. ``` Complete list of `SCXXXX` error codes: + +```text +add-default-case - Suggest adding a default case in `case` statements +avoid-nullary-conditions - Suggest explicitly using -n in `[ $var ]` +check-extra-masked-returns - Check for additional cases where exit codes are masked +check-set-e-suppressed - Notify when set -e is suppressed during function invocation +check-unassigned-uppercase - Warn when uppercase variables are unassigned +deprecate-which - Suggest 'command -v' instead of 'which' +quote-safe-variables - Suggest quoting variables without metacharacters +require-double-brackets - Require [[ and warn about [ in Bash/Ksh +require-variable-braces - Suggest putting braces around all variable references +``` + +Complete list of optional checks: + +```sh +# https://www.shellcheck.net/wiki/optional +shellcheck --list-optional +``` diff --git a/shellcheck/install.sh b/shellcheck/install.sh index 36984c2e9..d0a63c729 100644 --- a/shellcheck/install.sh +++ b/shellcheck/install.sh @@ -20,6 +20,10 @@ __init_shellcheck() { # pkg_install must be defined by every package pkg_install() { + if ! test -e ~/.shellcheckrc; then + touch ~/.shellcheckrc + fi + # ~/.local/opt/shellcheck-v0.99.9/bin mkdir -p "$(dirname "$pkg_src_cmd")" From 5544ff9f1b2fcd514f7de4fb1a374aa661698fa7 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Tue, 17 Dec 2024 20:10:23 +0000 Subject: [PATCH 22/27] feat(shellcheck): include ~/.shellcheckrc with example ignores and enables --- shellcheck/install.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/shellcheck/install.sh b/shellcheck/install.sh index d0a63c729..3889bc1ba 100644 --- a/shellcheck/install.sh +++ b/shellcheck/install.sh @@ -21,7 +21,13 @@ __init_shellcheck() { # pkg_install must be defined by every package pkg_install() { if ! test -e ~/.shellcheckrc; then - touch ~/.shellcheckrc + { + echo '# ignore: https://www.shellcheck.net/wiki/Ignore' + echo '# enable: https://www.shellcheck.net/wiki/optional' + echo '#disable=SC1090,SC1091' + echo '#enable=add-default-case,check-extra-masked-returns,deprecate-which' + echo '#enable=quote-safe-variables,check-set-e-suppressed,require-variable-braces' + } > ~/.shellcheckrc fi # ~/.local/opt/shellcheck-v0.99.9/bin From 910fa4827802ddd2b95a3ac7b25f031aabc7fc51 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Wed, 18 Dec 2024 07:22:11 +0000 Subject: [PATCH 23/27] doc(node): list node dependencies --- node/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/node/README.md b/node/README.md index 0b6c23a05..e65278221 100644 --- a/node/README.md +++ b/node/README.md @@ -363,3 +363,17 @@ jobs: - run: npm run lint - run: npm run test ``` + +### How to Install Node's Linux Dependencies + +Typically Node just needs `openssl` and `libstdc++`. + +```sh +# Apline +sudo apk add --no-cache libstdc++ libssl3 +``` + +```sh +# Debian / Ubuntu +sudo apt-get install -y libstdc++6 libssl3 +``` From afe35f9198b8cb656b5d3d86ca99866b7c86c794 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Wed, 18 Dec 2024 07:22:26 +0000 Subject: [PATCH 24/27] feat(node): ask to install libstdc++ on Alpine --- node/install.sh | 54 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/node/install.sh b/node/install.sh index 52573679a..117633b04 100644 --- a/node/install.sh +++ b/node/install.sh @@ -43,4 +43,58 @@ pkg_done_message() { b_dst="$(fn_sub_home "${pkg_dst}")" echo "" echo " Installed $(t_pkg 'node') and $(t_pkg 'npm') at $(t_path "${b_dst}/")" + + if command -v apk > /dev/null; then + if ! apk info | grep -F 'libstdc++' > /dev/null; then + echo "" + echo " $(t_pkg 'WARNING'): $(t_pkg 'libstdc++') is required for $(t_pkg 'node'), but not installed" >&2 + if command -v sudo > /dev/null; then + cmd_sudo='sudo ' + fi + _install_webi_essentials_apk "${cmd_sudo}" 'libstdc++' + fi + fi } + +_install_webi_essentials_apk() { ( + cmd_sudo="${1}" + b_pkgs="${2}" + + #echo " $(t_dim 'Running') $(t_cmd "${cmd_sudo}apk add --no-cache")" + fn_polite_sudo "${cmd_sudo}" " $(t_cmd "apk add --no-cache ${b_pkgs}")" + # shellcheck disable=SC2086 + ${cmd_sudo} apk add --no-cache ${b_pkgs} +); } + +fn_polite_sudo() { ( + a_sudo="${1}" + a_cmds="${2}" + + # no sudo needed, so don't ask + if test -z "${a_sudo}"; then + return 0 + fi + + # this is scripted, not user-interactive, continue + if test -z "${WEBI_TTY}"; then + return 0 + fi + + # this is user interactive, ask the user,defaulting to yes + echo "" + #shellcheck disable=SC2005 # echo for newline + printf '%s\n' "$(t_attn 'Use sudo to run the following? [Y/n] ')" + echo "${a_cmds}" + read -r b_yes < /dev/tty + + b_yes="$( + echo "${b_yes}" | + tr '[:upper:]' '[:lower:]' | + tr -d '[:space:]' + )" + if test -z "${b_yes}" || test "${b_yes}" = "y" || test "${b_yes}" = "yes"; then + return 0 + fi + echo " aborted" + return 1 +); } From cc66f930b0635e1abfcf07acbc088509db26fe2e Mon Sep 17 00:00:00 2001 From: OG Date: Wed, 20 Nov 2024 00:21:22 +0100 Subject: [PATCH 25/27] feat: add terramate --- terramate/README.md | 100 ++++++++++++++++++++++++++++++++++++++++++ terramate/install.ps1 | 56 +++++++++++++++++++++++ terramate/install.sh | 47 ++++++++++++++++++++ terramate/releases.js | 26 +++++++++++ 4 files changed, 229 insertions(+) create mode 100644 terramate/README.md create mode 100644 terramate/install.ps1 create mode 100644 terramate/install.sh create mode 100644 terramate/releases.js diff --git a/terramate/README.md b/terramate/README.md new file mode 100644 index 000000000..97c04513d --- /dev/null +++ b/terramate/README.md @@ -0,0 +1,100 @@ +--- +title: Terramate +homepage: https://github.com/terramate-io/terramate +tagline: | + Terramate simplifies managing large-scale Terraform codebases with a focus on automation and scalability. +--- + +To update or switch versions, run `webi terramate@stable` (or `@v1.0.0`, +`@beta`, etc). + +## Cheat Sheet + +The information in this section is a copy of the preflight requirements and +common command-line arguments from Terramate +(https://github.com/terramate-io/terramate). + +> `Terramate` enables scalable automation for Terraform by providing a robust +> framework for managing multiple stacks, generating code, and executing +> targeted workflows. + + +### **1. Create a New Project** +```bash +git init -b main terramate-quickstart +cd terramate-quickstart +git commit --allow-empty -m "Initial empty commit" +``` + +--- + +### **2. Create a Stack** +```bash +terramate create \ + --name "StackName" \ + --description "Description of the stack" \ + stacks/stackname + +git add stacks/stackname/stack.tm.hcl +git commit -m "Create a stack" +``` + +--- + +### **3. List Stacks** +```bash +terramate list +``` + +--- + +### **4. Detect Changes** +```bash +terramate list --changed +``` + +--- + +### **5. Generate Code** +1. Create a `.tm.hcl` file for code generation: + ```bash + cat <stacks/backend.tm.hcl + generate_hcl "backend.tf" { + content { + terraform { + backend "local" {} + } + } + } + EOF + ``` + +2. Run the generation command: + ```bash + terramate generate + ``` + +--- + +### **6. Run Terraform Commands** +- **Initialize stacks:** + ```bash + terramate run terraform init + ``` + +- **Plan changes:** + ```bash + terramate run terraform plan + ``` + +- **Apply changes:** + ```bash + terramate run terraform apply -auto-approve + ``` + +- **Run commands only on changed stacks:** + ```bash + terramate run --changed terraform init + terramate run --changed terraform plan + terramate run --changed terraform apply -auto-approve + ``` diff --git a/terramate/install.ps1 b/terramate/install.ps1 new file mode 100644 index 000000000..ca38f9ccf --- /dev/null +++ b/terramate/install.ps1 @@ -0,0 +1,56 @@ +#!/usr/bin/env pwsh + +############### +# Install terramate # +############### + +# Every package should define these variables +$pkg_cmd_name = "terramate" + +$pkg_dst_cmd = "$Env:USERPROFILE\.local\bin\terramate.exe" +$pkg_dst = "$pkg_dst_cmd" + +$pkg_src_cmd = "$Env:USERPROFILE\.local\opt\terramate-v$Env:WEBI_VERSION\bin\terramate.exe" +$pkg_src_bin = "$Env:USERPROFILE\.local\opt\terramate-v$Env:WEBI_VERSION\bin" +$pkg_src_dir = "$Env:USERPROFILE\.local\opt\terramate-v$Env:WEBI_VERSION" +$pkg_src = "$pkg_src_cmd" + +New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null +$pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE" + +# Fetch archive +IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) { + Write-Output "Downloading terramate from $Env:WEBI_PKG_URL to $pkg_download" + & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part" + & Move-Item "$pkg_download.part" "$pkg_download" +} + +IF (!(Test-Path -Path "$pkg_src_cmd")) { + Write-Output "Installing terramate" + + # TODO: create package-specific temp directory + # Enter tmp + Push-Location .local\tmp + + # Remove any leftover tmp cruft + Remove-Item -Path ".\terramate-v*" -Recurse -ErrorAction Ignore + Remove-Item -Path ".\terramate.exe" -Recurse -ErrorAction Ignore + + # Unpack archive file into this temporary directory + # Windows BSD-tar handles zip. Imagine that. + Write-Output "Unpacking $pkg_download" + & tar xf "$pkg_download" + + # Settle unpacked archive into place + Write-Output "Install Location: $pkg_src_cmd" + New-Item "$pkg_src_bin" -ItemType Directory -Force | Out-Null + Move-Item -Path "terramate.exe" -Destination "$pkg_src_bin" + + # Exit tmp + Pop-Location +} + +Write-Output "Copying into '$pkg_dst_cmd' from '$pkg_src_cmd'" +Remove-Item -Path "$pkg_dst_cmd" -Recurse -ErrorAction Ignore | Out-Null +Copy-Item -Path "$pkg_src" -Destination "$pkg_dst" -Recurse +Remove-Item -Path "$pkg_src" -Recurse -ErrorAction Ignore diff --git a/terramate/install.sh b/terramate/install.sh new file mode 100644 index 000000000..294e00b6e --- /dev/null +++ b/terramate/install.sh @@ -0,0 +1,47 @@ +#!/bin/sh +set -e +set -u + +__init_terramate() { + + ##################### + # Install terramate # + ##################### + + # Every package should define these 6 variables + pkg_cmd_name="terramate" + + pkg_dst_cmd="$HOME/.local/bin/terramate" + pkg_dst="$pkg_dst_cmd" + + pkg_src_cmd="$HOME/.local/opt/terramate-v$WEBI_VERSION/bin/terramate" + pkg_src_dir="$HOME/.local/opt/terramate-v$WEBI_VERSION" + pkg_src="$pkg_src_cmd" + + # pkg_install must be defined by every package + pkg_install() { + # ~/.local/opt/terramate-v0.99.9/bin + mkdir -p "$(dirname "$pkg_src_cmd")" + + # mv ./terramate-*/terramate ~/.local/opt/terramate-v0.99.9/bin/terramate + mv terramate "$pkg_src_cmd" + } + + # pkg_get_current_version is recommended, but (soon) not required + pkg_get_current_version() { + # 'terramate version' has output in this format: + + # 0.10.4 + + # Your version of Terramate is out of date! The latest version + # is 0.11.1 (released on Wed Oct 30 15:37:00 UTC 2024). + # You can update by downloading from https://github.com/mineiros-io/terramate/releases/tag/v0.11.1 + + # This trims it down to just the version number: + # 0.10.4 + terramate version | grep '^[0-9]\+\.[0-9]\+\.[0-9]\+$' + } + +} + +__init_terramate diff --git a/terramate/releases.js b/terramate/releases.js new file mode 100644 index 000000000..91d2e2ba5 --- /dev/null +++ b/terramate/releases.js @@ -0,0 +1,26 @@ +'use strict'; + +var github = require('../_common/github.js'); +var owner = 'terramate-io'; +var repo = 'terramate'; + +module.exports = function () { + return github(null, owner, repo).then(function (all) { + all.releases = all.releases.filter( + (release) => !['checksums.txt', 'cosign.pub'].includes(release.name), + ); + return all; + }); +}; + +if (module === require.main) { + module.exports().then(function (all) { + all = require('../_webi/normalize.js')(all); + // just select the first 5 for demonstration + // all.releases = all.releases.slice(0, 5); + // all.releases = all.releases.filter( + // release => !["checksums.txt.sig", "cosign.pub","terramate_0.9.0_windows_x86_64.zip"].includes(release.name) + // ); + console.info(JSON.stringify(all, null, 2)); + }); +} From 83a214a032c4e1cc873fa8be71fabd4820843ca0 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Wed, 18 Dec 2024 21:51:41 +0000 Subject: [PATCH 26/27] ref(terramate): mostly style updates --- terramate/README.md | 59 ++++++++++++++++++++----------------------- terramate/install.sh | 18 ++++++------- terramate/releases.js | 34 ++++++++++++++++--------- 3 files changed, 58 insertions(+), 53 deletions(-) diff --git a/terramate/README.md b/terramate/README.md index 97c04513d..7a1a3c335 100644 --- a/terramate/README.md +++ b/terramate/README.md @@ -2,63 +2,56 @@ title: Terramate homepage: https://github.com/terramate-io/terramate tagline: | - Terramate simplifies managing large-scale Terraform codebases with a focus on automation and scalability. + Terramate simplifies managing large-scale Terraform codebases --- -To update or switch versions, run `webi terramate@stable` (or `@v1.0.0`, +To update or switch versions, run `webi terramate@stable` (or `@v0.11.4`, `@beta`, etc). ## Cheat Sheet -The information in this section is a copy of the preflight requirements and -common command-line arguments from Terramate -(https://github.com/terramate-io/terramate). - > `Terramate` enables scalable automation for Terraform by providing a robust > framework for managing multiple stacks, generating code, and executing > targeted workflows. - ### **1. Create a New Project** -```bash -git init -b main terramate-quickstart -cd terramate-quickstart + +```sh +git init -b 'main' ./terramate-quickstart +cd ./terramate-quickstart git commit --allow-empty -m "Initial empty commit" ``` ---- - ### **2. Create a Stack** -```bash + +```sh terramate create \ --name "StackName" \ --description "Description of the stack" \ - stacks/stackname + ./stacks/stackname/ -git add stacks/stackname/stack.tm.hcl +git add ./stacks/stackname/stack.tm.hcl git commit -m "Create a stack" ``` ---- - ### **3. List Stacks** -```bash + +```sh terramate list ``` ---- - ### **4. Detect Changes** -```bash + +```sh terramate list --changed ``` ---- - ### **5. Generate Code** + 1. Create a `.tm.hcl` file for code generation: - ```bash - cat <stacks/backend.tm.hcl + + ```sh + cat < ./stacks/backend.tm.hcl generate_hcl "backend.tf" { content { terraform { @@ -70,30 +63,32 @@ terramate list --changed ``` 2. Run the generation command: - ```bash + ```sh terramate generate ``` ---- - ### **6. Run Terraform Commands** + - **Initialize stacks:** - ```bash + + ```sh terramate run terraform init ``` - **Plan changes:** - ```bash + + ```sh terramate run terraform plan ``` - **Apply changes:** - ```bash + + ```sh terramate run terraform apply -auto-approve ``` - **Run commands only on changed stacks:** - ```bash + ```sh terramate run --changed terraform init terramate run --changed terraform plan terramate run --changed terraform apply -auto-approve diff --git a/terramate/install.sh b/terramate/install.sh index 294e00b6e..1fc9bdcca 100644 --- a/terramate/install.sh +++ b/terramate/install.sh @@ -20,26 +20,26 @@ __init_terramate() { # pkg_install must be defined by every package pkg_install() { - # ~/.local/opt/terramate-v0.99.9/bin + # ~/.local/opt/terramate-v0.11.4/bin mkdir -p "$(dirname "$pkg_src_cmd")" - # mv ./terramate-*/terramate ~/.local/opt/terramate-v0.99.9/bin/terramate - mv terramate "$pkg_src_cmd" + # mv ./terramate* ~/.local/opt/terramate-v0.11.4/bin/terramate + mv terramate* "$pkg_src_cmd" } # pkg_get_current_version is recommended, but (soon) not required pkg_get_current_version() { # 'terramate version' has output in this format: - # 0.10.4 - + # 0.11.3 + # # Your version of Terramate is out of date! The latest version - # is 0.11.1 (released on Wed Oct 30 15:37:00 UTC 2024). - # You can update by downloading from https://github.com/mineiros-io/terramate/releases/tag/v0.11.1 + # is 0.11.4 (released on Tue Dec 3 19:27:35 UTC 2024). + # You can update by downloading from https://github.com/terramate-io/terramate/releases/tag/v0.11.4 # This trims it down to just the version number: - # 0.10.4 - terramate version | grep '^[0-9]\+\.[0-9]\+\.[0-9]\+$' + # 0.11.4 + terramate version | head -n 1 | cut -d' ' -f1 } } diff --git a/terramate/releases.js b/terramate/releases.js index 91d2e2ba5..c3d1c712c 100644 --- a/terramate/releases.js +++ b/terramate/releases.js @@ -1,20 +1,30 @@ 'use strict'; -var github = require('../_common/github.js'); -var owner = 'terramate-io'; -var repo = 'terramate'; +let github = require('../_common/github.js'); +let owner = 'terramate-io'; +let repo = 'terramate'; -module.exports = function () { - return github(null, owner, repo).then(function (all) { - all.releases = all.releases.filter( - (release) => !['checksums.txt', 'cosign.pub'].includes(release.name), - ); - return all; - }); -}; +let junkFiles = ['checksums.txt', 'cosign.pub']; + +async function getDistributables() { + let all = await github(null, owner, repo); + let releases = []; + for (let release of all.releases) { + let isJunk = !junkFiles.includes(release.name); + if (isJunk) { + continue; + } + releases.push(release); + } + + all.releases = releases; + return all; +} + +module.exports = getDistributables; if (module === require.main) { - module.exports().then(function (all) { + getDistributables().then(function (all) { all = require('../_webi/normalize.js')(all); // just select the first 5 for demonstration // all.releases = all.releases.slice(0, 5); From 976602236b434be17c218b39387496d99216a40b Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Wed, 18 Dec 2024 21:51:54 +0000 Subject: [PATCH 27/27] chore: npm run fmt --- caddy/README.md | 3 +-- node/README.md | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/caddy/README.md b/caddy/README.md index c6264fd68..a50142234 100644 --- a/caddy/README.md +++ b/caddy/README.md @@ -1181,8 +1181,7 @@ To prevent search engine and browser confusion - _DO NOT_ prevent crawling via `robots.txt` \ (counter-intuitive, but pages _must_ be crawled for links to _NOT_ be indexed) - _all_ domains using public TLS certs _will_ be indexed by default \ - (they are all linked to and crawled from various Certificate Transparency - reports) + (they are all linked to and crawled from various Certificate Transparency reports) - follow these guidelines even if the dev sites use HTTP Basic Auth ```Caddyfile diff --git a/node/README.md b/node/README.md index e65278221..7851c963f 100644 --- a/node/README.md +++ b/node/README.md @@ -66,8 +66,7 @@ To run them manually on your code; jhint -c ./.jshintrc *.js */*.js ``` - fixjson \ - (turns JavaScript Objects with comments, trailing commas, etc into actual - json) + (turns JavaScript Objects with comments, trailing commas, etc into actual json) ```sh fixjson -i 2 -w ./package.json ```