From 257ab726aaf3ba49fc5bbd2e9db0a24a0c462566 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 24 Apr 2024 15:26:18 +0200
Subject: [PATCH] Run the flake-regressions test suite

---
 .github/workflows/ci.yml     | 28 +++++++++++++++++-----------
 scripts/flake-regressions.sh | 27 +++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 11 deletions(-)
 create mode 100755 scripts/flake-regressions.sh

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ca94ff956..be7d47787 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -193,18 +193,24 @@ jobs:
       - uses: actions/checkout@v4
       - uses: DeterminateSystems/nix-installer-action@main
       - uses: DeterminateSystems/magic-nix-cache-action@main
-      - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes .#hydraJobs.tests.functional_user
+      - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes
 
-  meson_build:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-    runs-on: ${{ matrix.os }}
+  flake_regressions:
+    needs: vm_tests
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout nix
+        uses: actions/checkout@v4
+      - name: Checkout flake-regressions
+        uses: actions/checkout@v4
+        with:
+          repository: DeterminateSystems/flake-regressions
+          path: flake-regressions
+      - name: Checkout flake-regressions-data
+        uses: actions/checkout@v4
+        with:
+          repository: DeterminateSystems/flake-regressions-data
+          path: flake-regressions/tests
       - uses: DeterminateSystems/nix-installer-action@main
       - uses: DeterminateSystems/magic-nix-cache-action@main
-      # Only meson packages that don't have a tests.run derivation.
-      # Those that have it are already built and tested as part of nix flake check.
-      - run: nix build -L .#hydraJobs.build.{nix-cmd,nix-main}.$(nix-instantiate --eval --expr builtins.currentSystem | sed -e 's/"//g')
+      - run: nix build --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH scripts/flake-regressions.sh
diff --git a/scripts/flake-regressions.sh b/scripts/flake-regressions.sh
new file mode 100755
index 000000000..e6cfbfa24
--- /dev/null
+++ b/scripts/flake-regressions.sh
@@ -0,0 +1,27 @@
+#! /usr/bin/env bash
+
+set -e
+
+echo "Nix version:"
+nix --version
+
+cd flake-regressions
+
+status=0
+
+flakes=$(ls -d tests/*/*/* | head -n25)
+
+echo "Running flake tests..."
+
+for flake in $flakes; do
+
+    if ! REGENERATE=0 ./eval-flake.sh $flake; then
+        status=1
+        echo "❌ $flake"
+    else
+        echo "✅ $flake"
+    fi
+
+done
+
+exit "$status"