fix: broken tests (mostly out of stuff being outdated) (#468)

* Update rebar3's releases.json per current results

* Up rebar3 test versions

* Retire windows-2019

* Add "support" to win25-vs2026 next to win25

windows-2025-vs2026 label was previously a beta testing
tag but is now the production default

* Fix rebar3 test releases.json

* Bump per CI results

* Fix per CI results (I checked in releases.json)

* Adapt to Ubuntu 26.04 (no -arm)

* Consider Ubuntu ARM variants in execution

* Typo

* Bump it to the minimum expect value

* Bump it to the minimum expect value

* Tweak ARM tests: drop windows-11

* Fix per prior test results

* Try testing OTP to update README

* Improve guarantees on the README (via tests)

* Improve guarantees on the README (via tests)

Trust tests in intervals (if first Ok and last Ok the middle Ok)

* Add further constraints for testing (per README)

* Remove moving-target test

This is prone to fail for an issue with OTP's main
branch but it shouldn't affect the action, so we
remove it

* Respect Hex's boundaries

* Make it consistent (no need for .0 in versioning)

* Restrict it a bit

We keep the README as-is, since there's nothing preventing us
from thinking it works as stated, but we move our tests forward

* Make it easier to follow

* Fix it per stalled CI

* Fix it: no macos-16; it's macos-26

* Deprecate macos-13

Per https://github.blog/changelog/2025-07-11-upcoming-changes-to-macos-hosted-runners-macos-latest-migration-and-xcode-support-policy-updates/#macos-13-is-closing-down

* Make ImageOS support more explicit

* Move forward
This commit is contained in:
Paulo F. Oliveira
2026-06-28 15:53:20 +01:00
committed by GitHub
parent 3985c7c047
commit 6e9312bce0
9 changed files with 972 additions and 141 deletions
-1
View File
@@ -92,7 +92,6 @@ jobs:
strategy:
matrix:
os:
- windows-11-arm # ARM64
- windows-latest # x86_64
runs-on: ${{ matrix.os }}
steps:
+19 -11
View File
@@ -16,6 +16,7 @@ on:
jobs:
integration_test:
name: >
OS ${{matrix.combo.os}},
OTP ${{matrix.combo.otp-version}},
Elixir ${{matrix.combo.elixir-version}},
rebar3 ${{matrix.combo.rebar3-version}}
@@ -24,6 +25,22 @@ jobs:
fail-fast: false
matrix:
combo:
# macOS 14 - per README
- otp-version: '25'
os: 'macos-14'
- otp-version: '29'
os: 'macos-14'
# macOS 15 - per README
- otp-version: '25'
os: 'macos-15'
- otp-version: '29'
os: 'macos-15'
# macOS 26 - per README
- otp-version: '25'
os: 'macos-26'
- otp-version: '29'
os: 'macos-26'
# macOS latest
- otp-version: '27'
elixir-version: 'v1.17.0'
rebar3-version: '3.23'
@@ -43,22 +60,13 @@ jobs:
elixir-version: '1.14.3'
os: 'macos-latest'
version-type: 'strict'
- otp-version: '25'
os: 'macos-15'
- otp-version: '27'
elixir-version: '1.18'
os: 'macos-26'
- otp-version: '26'
elixir-version: '1.16'
rebar3-version: '3.25'
os: 'macos-14'
- otp-version: '25.0'
elixir-version: 'v1.13.4-otp-25'
elixir-version: 'v1.16.3-otp-25'
rebar3-version: '3.18.0'
os: 'macos-latest'
version-type: 'strict'
- otp-version: '25.0'
elixir-version: 'v1.13.4'
elixir-version: 'v1.16.3'
rebar3-version: '3.18.0'
os: 'macos-latest'
version-type: 'strict'
+17 -15
View File
@@ -16,6 +16,7 @@ on:
jobs:
integration_test:
name: >
OS ${{matrix.combo.os}},
OTP ${{matrix.combo.otp-version}},
Elixir ${{matrix.combo.elixir-version}},
Gleam ${{matrix.combo.gleam-version}},
@@ -25,15 +26,22 @@ jobs:
fail-fast: false
matrix:
combo:
- otp-version: '<26'
elixir-version: '<1.17'
# Ubuntu 22.04 - per README
- otp-version: '24.2'
os: 'ubuntu-22.04'
- otp-version: 'master'
elixir-version: '> 0'
- otp-version: '29'
os: 'ubuntu-22.04'
# Ubuntu 24.04 - per README
- otp-version: '24.3'
os: 'ubuntu-24.04'
- otp-version: 'master'
elixir-version: 'main'
- otp-version: '29'
os: 'ubuntu-24.04'
# Ubuntu 26.04 - per README
- otp-version: '26'
os: 'ubuntu-26.04'
- otp-version: '29'
os: 'ubuntu-26.04'
# Ubuntu latest
- otp-version: '27'
elixir-version: 'v1.17.0'
rebar3-version: '3.23'
@@ -53,19 +61,13 @@ jobs:
elixir-version: '1.14.3'
os: 'ubuntu-latest'
version-type: 'strict'
- otp-version: '25'
os: 'ubuntu-24.04'
- otp-version: '26'
elixir-version: '1.16'
rebar3-version: '3.25'
os: 'ubuntu-22.04'
- otp-version: '25.0'
elixir-version: 'v1.13.4-otp-25'
elixir-version: 'v1.16.2-otp-25'
rebar3-version: '3.18.0'
os: 'ubuntu-latest'
version-type: 'strict'
- otp-version: '25.0'
elixir-version: 'v1.13.4'
elixir-version: 'v1.16.3'
rebar3-version: '3.18.0'
os: 'ubuntu-latest'
version-type: 'strict'
@@ -80,7 +82,7 @@ jobs:
otp-version: false
os: 'ubuntu-latest'
disable_problem_matchers: true
- elixir-version: 'v1.13'
- elixir-version: 'v1.16'
otp-version: '24'
escript_packages: 'hex protobuf'
escript_script: 'protoc-gen-elixir --version'
+15 -12
View File
@@ -16,6 +16,7 @@ on:
jobs:
integration_test:
name: >
OS ${{matrix.combo.os}},
OTP ${{matrix.combo.otp-version}},
Elixir ${{matrix.combo.elixir-version}},
Gleam ${{matrix.combo.gleam-version}},
@@ -25,12 +26,20 @@ jobs:
fail-fast: false
matrix:
combo:
# Windows 2022 - per README
- otp-version: '26' # 21 is "so old", at this moment, we can't find it in latest 100
os: 'windows-2022'
- otp-version: '29'
os: 'windows-2022'
# Windows 2025 - per README
- otp-version: '26' # 21 is "so old", at this moment, we can't find it in latest 100
os: 'windows-2025'
- otp-version: '29'
os: 'windows-2025'
# Windows latest
- otp-version: '<26'
elixir-version: '<1.17'
os: 'ubuntu-22.04'
- otp-version: 'master'
elixir-version: '> 0'
os: 'ubuntu-24.04'
os: 'windows-latest'
- otp-version: '27'
elixir-version: 'v1.17.0'
rebar3-version: '3.23'
@@ -47,24 +56,18 @@ jobs:
elixir-version: '1.14.5'
os: 'windows-latest'
version-type: 'strict'
- otp-version: '24.0.2'
rebar3-version: '3.16'
os: 'windows-2022'
- otp-version: '23.0'
rebar3-version: '3.15'
os: 'windows-2022'
- elixir-version: 'v1.10'
otp-version: '23'
rebar3-version: '3.14'
os: 'windows-latest'
- elixir-version: 'v1.13'
- elixir-version: 'v1.16'
otp-version: '24'
rebar3-version: '3.15'
os: 'windows-latest'
- gleam-version: '1.9.0'
otp-version: '27'
os: 'windows-latest'
- elixir-version: 'v1.13'
- elixir-version: 'v1.16'
otp-version: '24'
escript_packages: 'hex protobuf'
escript_script: 'protoc-gen-elixir --version'
+13 -11
View File
@@ -80,15 +80,14 @@ and Erlang/OTP.
| Operating system | Erlang/OTP | OTP Architecture | Status
|- |- | - |-
| `ubuntu-22.04` | 24.2 - 28 | x86_64, arm64 | ✅
| `ubuntu-24.04` | 24.3 - 28 | x86_64, arm64 | ✅
| `windows-2019` | 21\* - 25 | x86_64, x86 | ✅
| `windows-2022` | 21\* - 28 | x86_64, x86 | ✅
| `windows-2025` | 21\* - 28 | x86_64, x86 | ✅
| `macOS-13` | 25.0 - 28 | x86_64, arm64 | ✅
| `macOS-14` | 25.0 - 28 | x86_64, arm64 | ✅
| `macOS-15` | 25.0 - 28 | x86_64, arm64 | ✅
| `macOS-26` | 25.0 - 28 | x86_64, arm64 | ✅
| `ubuntu-22.04` | 24.2 - 29 | x86_64, arm64 | ✅
| `ubuntu-24.04` | 24.3 - 29 | x86_64, arm64 | ✅
| `ubuntu-26.04` | 26 - 29 | x86_64, arm64 | ✅
| `windows-2022` | 21\* - 29 | x86_64, x86 | ✅
| `windows-2025` | 21\* - 29 | x86_64, x86 | ✅
| `macOS-14` | 25 - 29 | x86_64, arm64 | ✅
| `macOS-15` | 25 - 29 | x86_64, arm64 | ✅
| `macOS-26` | 25 - 29 | x86_64, arm64 | ✅
**Note** \*: prior to 23, Windows builds are only available for minor versions, e.g. 21.0, 21.3,
22.0, etc.
@@ -106,11 +105,14 @@ uses that to download assets:
| ImageOS | Operating system
|- |-
| `ubuntu22` | `ubuntu-22.04`
| `ubuntu22-arm64` | `ubuntu-22.04`
| `ubuntu24` | `ubuntu-24.04`
| `win19` | `windows-2019`
| `ubuntu24-arm64` | `ubuntu-24.04`
| `ubuntu26` | `ubuntu-26.04`
| `ubuntu26-arm64` | `ubuntu-26.04`
| `win22` | `windows-2022`
| `win25` | `windows-2025`
| `macos13` | `macOS-13`
| `win25-vs2026` | `windows-2025`
| `macos14` | `macOS-14`
| `macos15` | `macOS-15`
| `macos26` | `macOS-26`
+13 -3
View File
@@ -54975,7 +54975,7 @@ async function getElixirVersion(exSpec0) {
if (!foundCombo) {
throw new Error(
`Requested Elixir / Erlang/OTP version (${exSpec0} / tried ${otpVersionsMajor}) not ` +
'found in version list (did you check Compatibility between Elixir and Erlang/OTP?).' +
'found in version list (did you check Compatibility between Elixir and Erlang/OTP?). ' +
'Elixir and Erlang/OTP compatibility can be found at: ' +
'https://hexdocs.pm/elixir/compatibility-and-deprecations.html',
)
@@ -55355,13 +55355,21 @@ function getRunnerOSArchitecture() {
function getRunnerOSVersion() {
// List from https://github.com/actions/runner-images?tab=readme-ov-file#available-images
// at e065a395d52f491c58eb8fba8ab1dde2c7cfada1
// Not considered, at the moment (though it might be easy to adapt to...):
// - ubuntu-slim
// - all macOS intel, large, and xlarge variants
// - all win arm variants
const ImageOSToContainer = {
ubuntu22: 'ubuntu-22.04',
'ubuntu22-arm64': 'ubuntu-22.04',
ubuntu24: 'ubuntu-24.04',
win19: 'windows-2019',
'ubuntu24-arm64': 'ubuntu-24.04',
ubuntu26: 'ubuntu-26.04',
'ubuntu26-arm64': 'ubuntu-26.04',
win22: 'windows-2022',
win25: 'windows-2025',
macos13: 'macOS-13',
'win25-vs2026': 'windows-2025',
macos14: 'macOS-14',
macos15: 'macOS-15',
macos26: 'macOS-26',
@@ -55369,6 +55377,8 @@ function getRunnerOSVersion() {
const deprecatedImageOSToContainer = {
ubuntu18: 'ubuntu-18.04',
ubuntu20: 'ubuntu-20.04',
win19: 'windows-2019',
macos13: 'macOS-13',
}
const containerFromEnvImageOS = ImageOSToContainer[process.env.ImageOS]
if (!containerFromEnvImageOS) {
+13 -3
View File
@@ -280,7 +280,7 @@ async function getElixirVersion(exSpec0) {
if (!foundCombo) {
throw new Error(
`Requested Elixir / Erlang/OTP version (${exSpec0} / tried ${otpVersionsMajor}) not ` +
'found in version list (did you check Compatibility between Elixir and Erlang/OTP?).' +
'found in version list (did you check Compatibility between Elixir and Erlang/OTP?). ' +
'Elixir and Erlang/OTP compatibility can be found at: ' +
'https://hexdocs.pm/elixir/compatibility-and-deprecations.html',
)
@@ -661,13 +661,21 @@ function getRunnerOSArchitecture() {
function getRunnerOSVersion() {
// List from https://github.com/actions/runner-images?tab=readme-ov-file#available-images
// at e065a395d52f491c58eb8fba8ab1dde2c7cfada1
// Not considered, at the moment (though it might be easy to adapt to...):
// - ubuntu-slim
// - all macOS intel, large, and xlarge variants
// - all win arm variants
const ImageOSToContainer = {
ubuntu22: 'ubuntu-22.04',
'ubuntu22-arm64': 'ubuntu-22.04',
ubuntu24: 'ubuntu-24.04',
win19: 'windows-2019',
'ubuntu24-arm64': 'ubuntu-24.04',
ubuntu26: 'ubuntu-26.04',
'ubuntu26-arm64': 'ubuntu-26.04',
win22: 'windows-2022',
win25: 'windows-2025',
macos13: 'macOS-13',
'win25-vs2026': 'windows-2025',
macos14: 'macOS-14',
macos15: 'macOS-15',
macos26: 'macOS-26',
@@ -675,6 +683,8 @@ function getRunnerOSVersion() {
const deprecatedImageOSToContainer = {
ubuntu18: 'ubuntu-18.04',
ubuntu20: 'ubuntu-20.04',
win19: 'windows-2019',
macos13: 'macOS-13',
}
const containerFromEnvImageOS = ImageOSToContainer[process.env.ImageOS]
if (!containerFromEnvImageOS) {
+869 -72
View File
File diff suppressed because it is too large Load Diff
+11 -11
View File
@@ -273,7 +273,7 @@ describe('.getOTPVersion(_) - Erlang', () => {
assert.deepStrictEqual(got, expected)
spec = '23.0'
osVersion = 'windows-2019'
osVersion = 'windows-2022'
expected = '23.0.4'
got = await setupBeam.getOTPVersion(spec, osVersion)
assert.deepStrictEqual(got, expected)
@@ -294,7 +294,7 @@ describe('.getOTPVersion(_) - Erlang', () => {
assert.deepStrictEqual(got, expected)
spec = '23.0'
osVersion = 'windows-2019'
osVersion = 'windows-2022'
expected = '23.0.4'
got = await setupBeam.getOTPVersion(spec, osVersion)
assert.deepStrictEqual(got, expected)
@@ -625,9 +625,9 @@ describe('.getOTPVersion(_) - Elixir', () => {
assert.deepStrictEqual(got, expected)
before = simulateInput('version-type', 'strict')
spec = '1.17'
spec = '1.19'
otpVersion = 'master'
expected = 'v1.17-otp-27'
expected = 'v1.19-otp-28'
await setupBeam.installOTP(otpVersion)
got = await setupBeam.getElixirVersion(spec, otpVersion)
assert.deepStrictEqual(got, expected)
@@ -695,18 +695,18 @@ describe('.getOTPVersion(_) - rebar3', () => {
let spec
it('returns the expected value', async () => {
spec = '3.10.x'
expected = '3.10.0'
spec = '3.26.x'
expected = '3.26.0'
got = await setupBeam.getRebar3Version(spec)
assert.deepStrictEqual(got, expected)
spec = '3.11'
expected = '3.11.1'
spec = '3.25'
expected = '3.25.1'
got = await setupBeam.getRebar3Version(spec)
assert.deepStrictEqual(got, expected)
spec = '3.10'
expected = '3.10.0'
spec = '3.26'
expected = '3.26.0'
got = await setupBeam.getRebar3Version(spec)
assert.deepStrictEqual(got, expected)
})
@@ -972,7 +972,7 @@ describe('.getVersionFromSpec(_)', () => {
assert.deepStrictEqual(got, expected)
spec = 'latest'
expected = '3.23.0'
expected = '3.27.0'
got = setupBeam.getVersionFromSpec(spec, matrix.rebar3)
assert.deepStrictEqual(got, expected)
})