Feature tool-cache for Gleam and rebar3 (#223)

* Make iterating over mirrors more flexible (and reusable)

At the same time we fix an error, where we aren't using the
mirrors for the OTP build :)

* Make installation of cache-based tool callback -based

This hopefully eases introduction of new languages at the cost
of a bit of abstraction (there's an extensive comment on how to
fill in the install options' object)

We'll soon test this abstraction by making Gleam and rebar3
cached too

* Increase our visibility when debugging

* Improve on lessons learned to ease incorporation of further caches

(starting with Gleam on Linux)

* Cache Gleam on Windows

* Cache Rebar3 on Linux

Also, gets rid of all .sh

* Cache Rebar3 on Windows

Also, gets rid of all .ps1

* Act on CI results

And also be consistent when it comes to using
fs....Sync vs fs.promise....

* Move tests to a single file

* Move main executable to a single file

* Decrease number of changes to ease review

* Expose function required for tests

* Don't async/await when not required

* Resolve post- merge-conflict resolution issues
This commit is contained in:
Paulo F. Oliveira
2023-08-07 10:32:15 +01:00
committed by GitHub
parent 3ba5b86a2a
commit afb8586fc4
11 changed files with 936 additions and 838 deletions
+402 -82
View File
@@ -1,17 +1,18 @@
const core = require('@actions/core')
const { exec } = require('@actions/exec')
const tc = require('@actions/tool-cache')
const http = require('@actions/http-client')
const path = require('path')
const semver = require('semver')
const fs = require('fs')
const installer = require('./installer')
const os = require('os')
main().catch((err) => {
core.setFailed(err.message)
})
async function main() {
installer.checkPlatform()
checkPlatform()
const versionFilePath = getInput('version-file', false)
let versions
@@ -29,24 +30,16 @@ async function main() {
const elixirSpec = getInput('elixir-version', false, 'elixir', versions)
const gleamSpec = getInput('gleam-version', false, 'gleam', versions)
const rebar3Spec = getInput('rebar3-version', false, 'rebar', versions)
const hexMirrors = core.getMultilineInput('hexpm-mirrors', {
required: false,
})
if (otpSpec !== 'false') {
await installOTP(otpSpec, osVersion, hexMirrors)
const elixirInstalled = await maybeInstallElixir(
elixirSpec,
otpSpec,
hexMirrors,
)
await installOTP(otpSpec, osVersion)
const elixirInstalled = await maybeInstallElixir(elixirSpec, otpSpec)
if (elixirInstalled === true) {
const shouldMixRebar = getInput('install-rebar', false)
await mix(shouldMixRebar, 'rebar', hexMirrors)
await mix(shouldMixRebar, 'rebar')
const shouldMixHex = getInput('install-hex', false)
await mix(shouldMixHex, 'hex', hexMirrors)
await mix(shouldMixHex, 'hex')
}
} else if (!gleamSpec) {
throw new Error('otp-version=false is only available when installing Gleam')
@@ -56,38 +49,43 @@ async function main() {
await maybeInstallRebar3(rebar3Spec)
}
async function installOTP(otpSpec, osVersion, hexMirrors) {
const otpVersion = await getOTPVersion(otpSpec, osVersion, hexMirrors)
async function installOTP(otpSpec, osVersion) {
const otpVersion = await getOTPVersion(otpSpec, osVersion)
core.startGroup(`Installing Erlang/OTP ${otpVersion} - built on ${osVersion}`)
await installer.installOTP(osVersion, otpVersion, hexMirrors)
await doWithMirrors({
hexMirrors: hexMirrorsInput(),
actionTitle: `install Erlang/OTP ${otpVersion}`,
action: async (hexMirror) => {
await install('otp', {
osVersion,
toolVersion: otpVersion,
hexMirror,
})
},
})
core.setOutput('otp-version', otpVersion)
core.endGroup()
return otpVersion
}
async function maybeInstallElixir(elixirSpec, otpSpec, hexMirrors) {
async function maybeInstallElixir(elixirSpec, otpSpec) {
let installed = false
if (elixirSpec) {
const elixirVersion = await getElixirVersion(
elixirSpec,
otpSpec,
hexMirrors,
)
const elixirVersion = await getElixirVersion(elixirSpec, otpSpec)
core.startGroup(`Installing Elixir ${elixirVersion}`)
await installer.installElixir(elixirVersion, hexMirrors)
await doWithMirrors({
hexMirrors: hexMirrorsInput(),
actionTitle: `install Elixir ${elixirVersion}`,
action: async (hexMirror) => {
await install('elixir', {
toolVersion: elixirVersion,
hexMirror,
})
},
})
core.setOutput('elixir-version', elixirVersion)
const disableProblemMatchers = getInput('disable_problem_matchers', false)
if (disableProblemMatchers === 'false') {
const elixirMatchers = path.join(
__dirname,
'..',
'matchers',
'elixir-matchers.json',
)
core.info(`##[add-matcher]${elixirMatchers}`)
}
maybeEnableElixirProblemMatchers()
core.endGroup()
installed = true
@@ -96,30 +94,32 @@ async function maybeInstallElixir(elixirSpec, otpSpec, hexMirrors) {
return installed
}
async function mixWithMirrors(cmd, args, hexMirrors) {
if (hexMirrors.length === 0) {
throw new Error('mix failed with every mirror')
}
const [hexMirror, ...hexMirrorsT] = hexMirrors
process.env.HEX_MIRROR = hexMirror
try {
return await exec(cmd, args)
} catch (err) {
core.info(
`mix failed with mirror ${process.env.HEX_MIRROR} with message ${err.message})`,
function maybeEnableElixirProblemMatchers() {
const disableProblemMatchers = getInput('disable_problem_matchers', false)
if (disableProblemMatchers === 'false') {
const elixirMatchers = path.join(
__dirname,
'..',
'matchers',
'elixir-matchers.json',
)
core.info(`##[add-matcher]${elixirMatchers}`)
}
return mixWithMirrors(cmd, args, hexMirrorsT)
}
async function mix(shouldMix, what, hexMirrors) {
async function mix(shouldMix, what) {
if (shouldMix === 'true') {
const cmd = 'mix'
const args = [`local.${what}`, '--force']
core.startGroup(`Running ${cmd} ${args}`)
await mixWithMirrors(cmd, args, hexMirrors)
await doWithMirrors({
hexMirrors: hexMirrorsInput(),
actionTitle: `mix ${what}`,
action: async (hexMirror) => {
process.env.HEX_MIRROR = hexMirror
await exec(cmd, args)
},
})
core.endGroup()
}
}
@@ -129,7 +129,7 @@ async function maybeInstallGleam(gleamSpec) {
if (gleamSpec) {
const gleamVersion = await getGleamVersion(gleamSpec)
core.startGroup(`Installing Gleam ${gleamVersion}`)
await installer.installGleam(gleamVersion)
await install('gleam', { toolVersion: gleamVersion })
core.setOutput('gleam-version', gleamVersion)
core.addPath(`${process.env.RUNNER_TEMP}/.setup-beam/gleam/bin`)
core.endGroup()
@@ -150,7 +150,7 @@ async function maybeInstallRebar3(rebar3Spec) {
rebar3Version = await getRebar3Version(rebar3Spec)
}
core.startGroup(`Installing rebar3 ${rebar3Version}`)
await installer.installRebar3(rebar3Version)
await install('rebar3', { toolVersion: rebar3Version })
core.setOutput('rebar3-version', rebar3Version)
core.addPath(`${process.env.RUNNER_TEMP}/.setup-beam/rebar3/bin`)
core.endGroup()
@@ -161,9 +161,9 @@ async function maybeInstallRebar3(rebar3Spec) {
return installed
}
async function getOTPVersion(otpSpec0, osVersion, hexMirrors) {
const otpVersions = await getOTPVersions(osVersion, hexMirrors)
const spec = otpSpec0.replace(/^OTP-/, '')
async function getOTPVersion(otpSpec0, osVersion) {
const otpVersions = await getOTPVersions(osVersion)
let spec = otpSpec0.replace(/^OTP-/, '')
const versions = otpVersions
const otpVersion = getVersionFromSpec(spec, versions)
if (otpVersion === null) {
@@ -176,15 +176,15 @@ async function getOTPVersion(otpSpec0, osVersion, hexMirrors) {
return otpVersion // from the reference, for download
}
async function getElixirVersion(exSpec0, otpVersion0, hexMirrors) {
async function getElixirVersion(exSpec0, otpVersion0) {
const otpVersion = otpVersion0.match(/^([^-]+-)?(.+)$/)[2]
const otpVersionMajor = otpVersion.match(/^([^.]+).*$/)[1]
const [otpVersionsForElixirMap, elixirVersions] = await getElixirVersions(
hexMirrors,
)
const [otpVersionsForElixirMap, elixirVersions] = await getElixirVersions()
const spec = exSpec0.replace(/-otp-.*$/, '')
const versions = elixirVersions
const elixirVersionFromSpec = getVersionFromSpec(spec, versions)
if (elixirVersionFromSpec === null) {
throw new Error(
`Requested Elixir version (${exSpec0}) not found in version list ` +
@@ -248,12 +248,19 @@ async function getRebar3Version(r3Spec) {
return rebar3Version
}
async function getOTPVersions(osVersion, hexMirrors) {
async function getOTPVersions(osVersion) {
let otpVersionsListings
let originListing
if (process.platform === 'linux') {
originListing = `/builds/otp/${osVersion}/builds.txt`
otpVersionsListings = await getWithMirrors(originListing, hexMirrors)
otpVersionsListings = await doWithMirrors({
hexMirrors: hexMirrorsInput(),
actionTitle: `fetch ${originListing}`,
action: async (hexMirror) => {
const l = await get(`${hexMirror}${originListing}`, [null])
return l
},
})
} else if (process.platform === 'win32') {
originListing =
'https://api.github.com/repos/erlang/otp/releases?per_page=100'
@@ -296,11 +303,16 @@ async function getOTPVersions(osVersion, hexMirrors) {
return otpVersions
}
async function getElixirVersions(hexMirrors) {
const elixirVersionsListings = await getWithMirrors(
'/builds/elixir/builds.txt',
hexMirrors,
)
async function getElixirVersions() {
const originListing = '/builds/elixir/builds.txt'
const elixirVersionsListings = await doWithMirrors({
hexMirrors: hexMirrorsInput(),
actionTitle: `fetch ${originListing}`,
action: async (hexMirror) => {
const l = await get(`${hexMirror}${originListing}`, [null])
return l
},
})
const otpVersionsForElixirMap = {}
const elixirVersions = {}
@@ -428,6 +440,7 @@ function maybeCoerced(v) {
}
} catch {
// some stuff can't be coerced, like 'main'
core.debug(`Was not able to coerce ${v} with semver`)
ret = v
}
@@ -517,21 +530,6 @@ async function get(url0, pageIdxs) {
return Promise.all(pageIdxs.map(getPage))
}
async function getWithMirrors(resourcePath, hexMirrors) {
if (hexMirrors.length === 0) {
throw new Error(`Could not fetch ${resourcePath} from any hex.pm mirror`)
}
const [hexMirror, ...hexMirrorsT] = hexMirrors
try {
return await get(`${hexMirror}${resourcePath}`, [null])
} catch (err) {
core.info(`get failed for URL ${hexMirror}${resourcePath}`)
}
return getWithMirrors(resourcePath, hexMirrorsT)
}
function maybePrependWithV(v) {
if (isVersion(v)) {
return `v${v.replace('v', '')}`
@@ -652,11 +650,333 @@ function debugLog(groupName, message) {
)
}
function hexMirrorsInput() {
return core.getMultilineInput('hexpm-mirrors', {
required: false,
})
}
async function doWithMirrors(opts) {
const { hexMirrors, actionTitle, action } = opts
let actionRes
if (hexMirrors.length === 0) {
throw new Error(`Could not ${actionTitle} from any hex.pm mirror`)
}
const [hexMirror, ...hexMirrorsT] = hexMirrors
try {
actionRes = await action(hexMirror)
} catch (err) {
core.info(
`Action ${actionTitle} failed for mirror ${hexMirror}, with ${err}`,
)
core.debug(`Stacktrace: ${err.stack}`)
actionRes = await doWithMirrors({
hexMirrors: hexMirrorsT,
actionTitle,
action,
})
}
return actionRes
}
async function install(toolName, opts) {
const { osVersion, toolVersion, hexMirror } = opts
const versionSpec =
osVersion !== undefined ? `${osVersion}/${toolVersion}` : toolVersion
let installOpts
// The installOpts object is composed of supported processPlatform keys
// (e.g. 'linux', 'win32', or 'all' - in case there's no distinction between platforms)
// In each of these keys there's an object with keys:
// * downloadToolURL
// - where to fetch the downloadable from
// * extract
// - if the downloadable is compressed: how to extract it
// - return ['dir', targetDir]
// - if the downloadable is not compressed: a filename.ext you want to cache it under
// - return ['file', filenameWithExt]
// * postExtract
// - stuff to execute outside the cache scope (just after it's created)
// * reportVersion
/// - configuration elements on how to output the tool version, post-install
switch (toolName) {
case 'otp':
installOpts = {
tool: 'Erlang/OTP',
linux: {
downloadToolURL: () =>
`${hexMirror}/builds/otp/${versionSpec}.tar.gz`,
extract: async (file) => {
const dest = undefined
const flags = ['zx', '--strip-components=1']
const targetDir = await tc.extractTar(file, dest, flags)
return ['dir', targetDir]
},
postExtract: async (cachePath) => {
const cmd = path.join(cachePath, 'Install')
const args = ['-minimal', cachePath]
await exec(cmd, args)
},
reportVersion: () => {
const cmd = 'erl'
const args = ['-version']
return [cmd, args]
},
},
win32: {
downloadToolURL: () =>
'https://github.com/erlang/otp/releases/download/' +
`OTP-${toolVersion}/otp_win64_${toolVersion}.exe`,
extract: async () => ['file', 'otp.exe'],
postExtract: async (cachePath) => {
const cmd = path.join(cachePath, 'otp.exe')
const args = ['/S', `/D=${cachePath}`]
await exec(cmd, args)
},
reportVersion: () => {
const cmd = 'erl.exe'
const args = ['+V']
return [cmd, args]
},
},
}
break
case 'elixir':
installOpts = {
tool: 'Elixir',
all: {
downloadToolURL: () =>
`${hexMirror}/builds/elixir/${versionSpec}.zip`,
extract: async (file) => {
const targetDir = await tc.extractZip(file)
return ['dir', targetDir]
},
postExtract: async () => {
const escriptsPath = path.join(os.homedir(), '.mix', 'escripts')
fs.mkdirSync(escriptsPath, { recursive: true })
core.addPath(escriptsPath)
if (debugLoggingEnabled()) {
core.exportVariable('ELIXIR_CLI_ECHO', 'true')
}
},
reportVersion: () => {
const cmd = 'elixir'
const args = ['-v']
return [cmd, args]
},
},
}
break
case 'gleam':
installOpts = {
tool: 'Gleam',
linux: {
downloadToolURL: () => {
let gz
if (
versionSpec === 'nightly' ||
semver.gt(versionSpec, 'v0.22.1')
) {
gz = `gleam-${versionSpec}-x86_64-unknown-linux-musl.tar.gz`
} else {
gz = `gleam-${versionSpec}-linux-amd64.tar.gz`
}
return `https://github.com/gleam-lang/gleam/releases/download/${versionSpec}/${gz}`
},
extract: async (file) => {
const dest = undefined
const flags = ['zx']
const targetDir = await tc.extractTar(file, dest, flags)
return ['dir', targetDir]
},
postExtract: async (cachePath) => {
const bindir = path.join(cachePath, 'bin')
const oldPath = path.join(cachePath, 'gleam')
const newPath = path.join(bindir, 'gleam')
fs.mkdirSync(bindir)
fs.renameSync(oldPath, newPath)
},
reportVersion: () => {
const cmd = 'gleam'
const args = ['--version']
return [cmd, args]
},
},
win32: {
downloadToolURL: () => {
let zip
if (
versionSpec === 'nightly' ||
semver.gt(versionSpec, 'v0.22.1')
) {
zip = `gleam-${versionSpec}-x86_64-pc-windows-msvc.zip`
} else {
zip = `gleam-${versionSpec}-windows-64bit.zip`
}
return `https://github.com/gleam-lang/gleam/releases/download/${versionSpec}/${zip}`
},
extract: async (file) => {
const targetDir = await tc.extractZip(file)
return ['dir', targetDir]
},
postExtract: async (cachePath) => {
const bindir = path.join(cachePath, 'bin')
const oldPath = path.join(cachePath, 'gleam.exe')
const newPath = path.join(bindir, 'gleam.exe')
fs.mkdirSync(bindir)
fs.renameSync(oldPath, newPath)
},
reportVersion: () => {
const cmd = 'gleam.exe'
const args = ['--version']
return [cmd, args]
},
},
}
break
case 'rebar3':
installOpts = {
tool: 'Rebar3',
linux: {
downloadToolURL: () => {
let url
if (versionSpec === 'nightly') {
url = 'https://s3.amazonaws.com/rebar3-nightly/rebar3'
} else {
url = `https://github.com/erlang/rebar3/releases/download/${versionSpec}/rebar3`
}
return url
},
extract: async () => ['file', 'rebar3'],
postExtract: async (cachePath) => {
const bindir = path.join(cachePath, 'bin')
const oldPath = path.join(cachePath, 'rebar3')
const newPath = path.join(bindir, 'rebar3')
fs.mkdirSync(bindir)
fs.renameSync(oldPath, newPath)
fs.chmodSync(newPath, 0o755)
},
reportVersion: () => {
const cmd = 'rebar3'
const args = ['version']
return [cmd, args]
},
},
win32: {
downloadToolURL: () => {
let url
if (versionSpec === 'nightly') {
url = 'https://s3.amazonaws.com/rebar3-nightly/rebar3'
} else {
url = `https://github.com/erlang/rebar3/releases/download/${versionSpec}/rebar3`
}
return url
},
extract: async () => ['file', 'rebar3'],
postExtract: async (cachePath) => {
const bindir = path.join(cachePath, 'bin')
const oldPath = path.join(cachePath, 'rebar3')
fs.mkdirSync(bindir)
fs.chmodSync(oldPath, 0o755)
const ps1Filename = path.join(bindir, 'rebar3.ps1')
fs.writeFileSync(ps1Filename, `& escript.exe ${oldPath} \${args}`)
const cmdFilename = path.join(bindir, 'rebar3.cmd')
fs.writeFileSync(
cmdFilename,
`@echo off\r\nescript.exe ${oldPath} %*`,
)
},
reportVersion: () => {
const cmd = 'rebar3.cmd'
const args = ['version']
return [cmd, args]
},
},
}
break
default:
throw new Error(`no installer for ${toolName}`)
}
await installTool({ toolName, versionSpec, installOpts })
}
async function installTool(opts) {
const { toolName, versionSpec, installOpts } = opts
const platformOpts = installOpts[process.platform] || installOpts.all
let cachePath = tc.find(toolName, versionSpec)
core.debug(`Checking if ${installOpts.tool} is already cached...`)
if (cachePath === '') {
core.debug(" ... it isn't!")
const downloadToolURL = platformOpts.downloadToolURL()
const file = await tc.downloadTool(downloadToolURL)
const [targetElemType, targetElem] = await platformOpts.extract(file)
if (targetElemType === 'dir') {
cachePath = await tc.cacheDir(targetElem, toolName, versionSpec)
} else if (targetElemType === 'file') {
cachePath = await tc.cacheFile(file, targetElem, toolName, versionSpec)
}
} else {
core.debug(` ... it is, at ${cachePath}`)
}
core.debug('Performing post extract operations...')
await platformOpts.postExtract(cachePath)
core.debug(`Adding ${cachePath}'s bin to system path`)
core.addPath(path.join(cachePath, 'bin'))
const installDirForVarName = `INSTALL_DIR_FOR_${toolName}`.toUpperCase()
core.debug(`Exporting ${installDirForVarName} as ${cachePath}`)
core.exportVariable(installDirForVarName, cachePath)
core.info(`Installed ${installOpts.tool} version`)
const [cmd, args] = platformOpts.reportVersion()
await exec(cmd, args)
}
function checkPlatform() {
if (process.platform !== 'linux' && process.platform !== 'win32') {
throw new Error(
'@erlef/setup-beam only supports Ubuntu and Windows at this time',
)
}
}
function debugLoggingEnabled() {
return !!process.env.RUNNER_DEBUG
}
module.exports = {
getOTPVersion,
getElixirVersion,
getGleamVersion,
getRebar3Version,
getVersionFromSpec,
install,
parseVersionFile,
}