fix: broken tests (mostly out of stuff being outdated) (#468)

* Update rebar3's releases.json per current results

* Up rebar3 test versions

* Retire windows-2019

* Add "support" to win25-vs2026 next to win25

windows-2025-vs2026 label was previously a beta testing
tag but is now the production default

* Fix rebar3 test releases.json

* Bump per CI results

* Fix per CI results (I checked in releases.json)

* Adapt to Ubuntu 26.04 (no -arm)

* Consider Ubuntu ARM variants in execution

* Typo

* Bump it to the minimum expect value

* Bump it to the minimum expect value

* Tweak ARM tests: drop windows-11

* Fix per prior test results

* Try testing OTP to update README

* Improve guarantees on the README (via tests)

* Improve guarantees on the README (via tests)

Trust tests in intervals (if first Ok and last Ok the middle Ok)

* Add further constraints for testing (per README)

* Remove moving-target test

This is prone to fail for an issue with OTP's main
branch but it shouldn't affect the action, so we
remove it

* Respect Hex's boundaries

* Make it consistent (no need for .0 in versioning)

* Restrict it a bit

We keep the README as-is, since there's nothing preventing us
from thinking it works as stated, but we move our tests forward

* Make it easier to follow

* Fix it per stalled CI

* Fix it: no macos-16; it's macos-26

* Deprecate macos-13

Per https://github.blog/changelog/2025-07-11-upcoming-changes-to-macos-hosted-runners-macos-latest-migration-and-xcode-support-policy-updates/#macos-13-is-closing-down

* Make ImageOS support more explicit

* Move forward

.github/workflows/action.yml

@@ -92,7 +92,6 @@ jobs:
     strategy:
       matrix:
         os:
-          - windows-11-arm # ARM64
           - windows-latest # x86_64
     runs-on: ${{ matrix.os }}
     steps:

.github/workflows/macos.yml

@@ -16,6 +16,7 @@ on:
 jobs:
   integration_test:
     name: >
+      OS ${{matrix.combo.os}},
       OTP ${{matrix.combo.otp-version}},
       Elixir ${{matrix.combo.elixir-version}},
       rebar3 ${{matrix.combo.rebar3-version}}
@@ -24,6 +25,22 @@ jobs:
       fail-fast: false
       matrix:
         combo:
+          # macOS 14 - per README
+          - otp-version: '25'
+            os: 'macos-14'
+          - otp-version: '29'
+            os: 'macos-14'
+          # macOS 15 - per README
+          - otp-version: '25'
+            os: 'macos-15'
+          - otp-version: '29'
+            os: 'macos-15'
+          # macOS 26 - per README
+          - otp-version: '25'
+            os: 'macos-26'
+          - otp-version: '29'
+            os: 'macos-26'
+          # macOS latest
           - otp-version: '27'
             elixir-version: 'v1.17.0'
             rebar3-version: '3.23'
@@ -43,22 +60,13 @@ jobs:
             elixir-version: '1.14.3'
             os: 'macos-latest'
             version-type: 'strict'
-          - otp-version: '25'
-            os: 'macos-15'
-          - otp-version: '27'
-            elixir-version: '1.18'
-            os: 'macos-26'
-          - otp-version: '26'
-            elixir-version: '1.16'
-            rebar3-version: '3.25'
-            os: 'macos-14'
           - otp-version: '25.0'
-            elixir-version: 'v1.13.4-otp-25'
+            elixir-version: 'v1.16.3-otp-25'
             rebar3-version: '3.18.0'
             os: 'macos-latest'
             version-type: 'strict'
           - otp-version: '25.0'
-            elixir-version: 'v1.13.4'
+            elixir-version: 'v1.16.3'
             rebar3-version: '3.18.0'
             os: 'macos-latest'
             version-type: 'strict'

.github/workflows/ubuntu.yml

@@ -16,6 +16,7 @@ on:
 jobs:
   integration_test:
     name: >
+      OS ${{matrix.combo.os}},
       OTP ${{matrix.combo.otp-version}},
       Elixir ${{matrix.combo.elixir-version}},
       Gleam ${{matrix.combo.gleam-version}},
@@ -25,15 +26,22 @@ jobs:
       fail-fast: false
       matrix:
         combo:
-          - otp-version: '<26'
+          # Ubuntu 22.04 - per README
-            elixir-version: '<1.17'
+          - otp-version: '24.2'
             os: 'ubuntu-22.04'
-          - otp-version: 'master'
+          - otp-version: '29'
-            elixir-version: '> 0'
+            os: 'ubuntu-22.04'
+          # Ubuntu 24.04 - per README
+          - otp-version: '24.3'
             os: 'ubuntu-24.04'
-          - otp-version: 'master'
+          - otp-version: '29'
-            elixir-version: 'main'
             os: 'ubuntu-24.04'
+          # Ubuntu 26.04 - per README
+          - otp-version: '26'
+            os: 'ubuntu-26.04'
+          - otp-version: '29'
+            os: 'ubuntu-26.04'
+          # Ubuntu latest
           - otp-version: '27'
             elixir-version: 'v1.17.0'
             rebar3-version: '3.23'
@@ -53,19 +61,13 @@ jobs:
             elixir-version: '1.14.3'
             os: 'ubuntu-latest'
             version-type: 'strict'
-          - otp-version: '25'
-            os: 'ubuntu-24.04'
-          - otp-version: '26'
-            elixir-version: '1.16'
-            rebar3-version: '3.25'
-            os: 'ubuntu-22.04'
           - otp-version: '25.0'
-            elixir-version: 'v1.13.4-otp-25'
+            elixir-version: 'v1.16.2-otp-25'
             rebar3-version: '3.18.0'
             os: 'ubuntu-latest'
             version-type: 'strict'
           - otp-version: '25.0'
-            elixir-version: 'v1.13.4'
+            elixir-version: 'v1.16.3'
             rebar3-version: '3.18.0'
             os: 'ubuntu-latest'
             version-type: 'strict'
@@ -80,7 +82,7 @@ jobs:
             otp-version: false
             os: 'ubuntu-latest'
             disable_problem_matchers: true
-          - elixir-version: 'v1.13'
+          - elixir-version: 'v1.16'
             otp-version: '24'
             escript_packages: 'hex protobuf'
             escript_script: 'protoc-gen-elixir --version'

.github/workflows/windows.yml

@@ -16,6 +16,7 @@ on:
 jobs:
   integration_test:
     name: >
+      OS ${{matrix.combo.os}},
       OTP ${{matrix.combo.otp-version}},
       Elixir ${{matrix.combo.elixir-version}},
       Gleam ${{matrix.combo.gleam-version}},
@@ -25,12 +26,20 @@ jobs:
       fail-fast: false
       matrix:
         combo:
+          # Windows 2022 - per README
+          - otp-version: '26' # 21 is "so old", at this moment, we can't find it in latest 100
+            os: 'windows-2022'
+          - otp-version: '29'
+            os: 'windows-2022'
+          # Windows 2025 - per README
+          - otp-version: '26' # 21 is "so old", at this moment, we can't find it in latest 100
+            os: 'windows-2025'
+          - otp-version: '29'
+            os: 'windows-2025'
+          # Windows latest
           - otp-version: '<26'
             elixir-version: '<1.17'
-            os: 'ubuntu-22.04'
+            os: 'windows-latest'
-          - otp-version: 'master'
-            elixir-version: '> 0'
-            os: 'ubuntu-24.04'
           - otp-version: '27'
             elixir-version: 'v1.17.0'
             rebar3-version: '3.23'
@@ -47,24 +56,18 @@ jobs:
             elixir-version: '1.14.5'
             os: 'windows-latest'
             version-type: 'strict'
-          - otp-version: '24.0.2'
-            rebar3-version: '3.16'
-            os: 'windows-2022'
-          - otp-version: '23.0'
-            rebar3-version: '3.15'
-            os: 'windows-2022'
           - elixir-version: 'v1.10'
             otp-version: '23'
             rebar3-version: '3.14'
             os: 'windows-latest'
-          - elixir-version: 'v1.13'
+          - elixir-version: 'v1.16'
             otp-version: '24'
             rebar3-version: '3.15'
             os: 'windows-latest'
           - gleam-version: '1.9.0'
             otp-version: '27'
             os: 'windows-latest'
-          - elixir-version: 'v1.13'
+          - elixir-version: 'v1.16'
             otp-version: '24'
             escript_packages: 'hex protobuf'
             escript_script: 'protoc-gen-elixir --version'

README.md

@@ -78,17 +78,16 @@ expect to be considered the latest.
 This list presents the known working version combos between the target operating system
 and Erlang/OTP.
 
-| Operating system | Erlang/OTP   | OTP Architecture | Status
+| Operating system | Erlang/OTP | OTP Architecture | Status
-|-                 |-             | -                |-
+|-                 |-           | -                |-
-| `ubuntu-22.04`   | 24.2 - 28    | x86_64, arm64    | ✅
+| `ubuntu-22.04`   | 24.2 - 29  | x86_64, arm64    | ✅
-| `ubuntu-24.04`   | 24.3 - 28    | x86_64, arm64    | ✅
+| `ubuntu-24.04`   | 24.3 - 29  | x86_64, arm64    | ✅
-| `windows-2019`   | 21\* - 25    | x86_64, x86      | ✅
+| `ubuntu-26.04`   | 26 - 29    | x86_64, arm64    | ✅
-| `windows-2022`   | 21\* - 28    | x86_64, x86      | ✅
+| `windows-2022`   | 21\* - 29  | x86_64, x86      | ✅
-| `windows-2025`   | 21\* - 28    | x86_64, x86      | ✅
+| `windows-2025`   | 21\* - 29  | x86_64, x86      | ✅
-| `macOS-13`       | 25.0 - 28    | x86_64, arm64    | ✅
+| `macOS-14`       | 25 - 29    | x86_64, arm64    | ✅
-| `macOS-14`       | 25.0 - 28    | x86_64, arm64    | ✅
+| `macOS-15`       | 25 - 29    | x86_64, arm64    | ✅
-| `macOS-15`       | 25.0 - 28    | x86_64, arm64    | ✅
+| `macOS-26`       | 25 - 29    | x86_64, arm64    | ✅
-| `macOS-26`       | 25.0 - 28    | x86_64, arm64    | ✅
 
 **Note** \*: prior to 23, Windows builds are only available for minor versions, e.g. 21.0, 21.3,
 22.0, etc.
@@ -106,11 +105,14 @@ uses that to download assets:
 | ImageOS            | Operating system
 |-                   |-
 | `ubuntu22`         | `ubuntu-22.04`
+| `ubuntu22-arm64`   | `ubuntu-22.04`
 | `ubuntu24`         | `ubuntu-24.04`
-| `win19`            | `windows-2019`
+| `ubuntu24-arm64`   | `ubuntu-24.04`
+| `ubuntu26`         | `ubuntu-26.04`
+| `ubuntu26-arm64`   | `ubuntu-26.04`
 | `win22`            | `windows-2022`
 | `win25`            | `windows-2025`
-| `macos13`          | `macOS-13`
+| `win25-vs2026`     | `windows-2025`
 | `macos14`          | `macOS-14`
 | `macos15`          | `macOS-15`
 | `macos26`          | `macOS-26`

dist/index.js

@@ -54975,7 +54975,7 @@ async function getElixirVersion(exSpec0) {
   if (!foundCombo) {
     throw new Error(
       `Requested Elixir / Erlang/OTP version (${exSpec0} / tried ${otpVersionsMajor}) not ` +
-        'found in version list (did you check Compatibility between Elixir and Erlang/OTP?).' +
+        'found in version list (did you check Compatibility between Elixir and Erlang/OTP?). ' +
         'Elixir and Erlang/OTP compatibility can be found at: ' +
         'https://hexdocs.pm/elixir/compatibility-and-deprecations.html',
     )
@@ -55355,13 +55355,21 @@ function getRunnerOSArchitecture() {
 
 function getRunnerOSVersion() {
   // List from https://github.com/actions/runner-images?tab=readme-ov-file#available-images
+  // at e065a395d52f491c58eb8fba8ab1dde2c7cfada1
+  // Not considered, at the moment (though it might be easy to adapt to...):
+  // - ubuntu-slim
+  // - all macOS intel, large, and xlarge variants
+  // - all win arm variants
   const ImageOSToContainer = {
     ubuntu22: 'ubuntu-22.04',
+    'ubuntu22-arm64': 'ubuntu-22.04',
     ubuntu24: 'ubuntu-24.04',
-    win19: 'windows-2019',
+    'ubuntu24-arm64': 'ubuntu-24.04',
+    ubuntu26: 'ubuntu-26.04',
+    'ubuntu26-arm64': 'ubuntu-26.04',
     win22: 'windows-2022',
     win25: 'windows-2025',
-    macos13: 'macOS-13',
+    'win25-vs2026': 'windows-2025',
     macos14: 'macOS-14',
     macos15: 'macOS-15',
     macos26: 'macOS-26',
@@ -55369,6 +55377,8 @@ function getRunnerOSVersion() {
   const deprecatedImageOSToContainer = {
     ubuntu18: 'ubuntu-18.04',
     ubuntu20: 'ubuntu-20.04',
+    win19: 'windows-2019',
+    macos13: 'macOS-13',
   }
   const containerFromEnvImageOS = ImageOSToContainer[process.env.ImageOS]
   if (!containerFromEnvImageOS) {

src/setup-beam.js

@@ -280,7 +280,7 @@ async function getElixirVersion(exSpec0) {
   if (!foundCombo) {
     throw new Error(
       `Requested Elixir / Erlang/OTP version (${exSpec0} / tried ${otpVersionsMajor}) not ` +
-        'found in version list (did you check Compatibility between Elixir and Erlang/OTP?).' +
+        'found in version list (did you check Compatibility between Elixir and Erlang/OTP?). ' +
         'Elixir and Erlang/OTP compatibility can be found at: ' +
         'https://hexdocs.pm/elixir/compatibility-and-deprecations.html',
     )
@@ -661,13 +661,21 @@ function getRunnerOSArchitecture() {
 
 function getRunnerOSVersion() {
   // List from https://github.com/actions/runner-images?tab=readme-ov-file#available-images
+  // at e065a395d52f491c58eb8fba8ab1dde2c7cfada1
+  // Not considered, at the moment (though it might be easy to adapt to...):
+  // - ubuntu-slim
+  // - all macOS intel, large, and xlarge variants
+  // - all win arm variants
   const ImageOSToContainer = {
     ubuntu22: 'ubuntu-22.04',
+    'ubuntu22-arm64': 'ubuntu-22.04',
     ubuntu24: 'ubuntu-24.04',
-    win19: 'windows-2019',
+    'ubuntu24-arm64': 'ubuntu-24.04',
+    ubuntu26: 'ubuntu-26.04',
+    'ubuntu26-arm64': 'ubuntu-26.04',
     win22: 'windows-2022',
     win25: 'windows-2025',
-    macos13: 'macOS-13',
+    'win25-vs2026': 'windows-2025',
     macos14: 'macOS-14',
     macos15: 'macOS-15',
     macos26: 'macOS-26',
@@ -675,6 +683,8 @@ function getRunnerOSVersion() {
   const deprecatedImageOSToContainer = {
     ubuntu18: 'ubuntu-18.04',
     ubuntu20: 'ubuntu-20.04',
+    win19: 'windows-2019',
+    macos13: 'macOS-13',
   }
   const containerFromEnvImageOS = ImageOSToContainer[process.env.ImageOS]
   if (!containerFromEnvImageOS) {

test/setup-beam.test.js

@@ -273,7 +273,7 @@ describe('.getOTPVersion(_) - Erlang', () => {
       assert.deepStrictEqual(got, expected)
 
       spec = '23.0'
-      osVersion = 'windows-2019'
+      osVersion = 'windows-2022'
       expected = '23.0.4'
       got = await setupBeam.getOTPVersion(spec, osVersion)
       assert.deepStrictEqual(got, expected)
@@ -294,7 +294,7 @@ describe('.getOTPVersion(_) - Erlang', () => {
       assert.deepStrictEqual(got, expected)
 
       spec = '23.0'
-      osVersion = 'windows-2019'
+      osVersion = 'windows-2022'
       expected = '23.0.4'
       got = await setupBeam.getOTPVersion(spec, osVersion)
       assert.deepStrictEqual(got, expected)
@@ -625,9 +625,9 @@ describe('.getOTPVersion(_) - Elixir', () => {
       assert.deepStrictEqual(got, expected)
 
       before = simulateInput('version-type', 'strict')
-      spec = '1.17'
+      spec = '1.19'
       otpVersion = 'master'
-      expected = 'v1.17-otp-27'
+      expected = 'v1.19-otp-28'
       await setupBeam.installOTP(otpVersion)
       got = await setupBeam.getElixirVersion(spec, otpVersion)
       assert.deepStrictEqual(got, expected)
@@ -695,18 +695,18 @@ describe('.getOTPVersion(_) - rebar3', () => {
   let spec
 
   it('returns the expected value', async () => {
-    spec = '3.10.x'
+    spec = '3.26.x'
-    expected = '3.10.0'
+    expected = '3.26.0'
     got = await setupBeam.getRebar3Version(spec)
     assert.deepStrictEqual(got, expected)
 
-    spec = '3.11'
+    spec = '3.25'
-    expected = '3.11.1'
+    expected = '3.25.1'
     got = await setupBeam.getRebar3Version(spec)
     assert.deepStrictEqual(got, expected)
 
-    spec = '3.10'
+    spec = '3.26'
-    expected = '3.10.0'
+    expected = '3.26.0'
     got = await setupBeam.getRebar3Version(spec)
     assert.deepStrictEqual(got, expected)
   })
@@ -972,7 +972,7 @@ describe('.getVersionFromSpec(_)', () => {
     assert.deepStrictEqual(got, expected)
 
     spec = 'latest'
-    expected = '3.23.0'
+    expected = '3.27.0'
     got = setupBeam.getVersionFromSpec(spec, matrix.rebar3)
     assert.deepStrictEqual(got, expected)
   })