installer: make sure we can always test the installer in ci and locally

Just now there is a dependency on cachix, which means we cannot test
the installer in CI if forks do not have the necessary secrets set up.
We replace this with a simple http server that serves the installer and
can be both used in CI and locally.

.github/workflows/ci.yml

@@ -17,7 +17,6 @@ jobs:
     - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json
 
   tests:
-    needs: [check_secrets]
     strategy:
       fail-fast: false
       matrix:
@@ -34,72 +33,21 @@ jobs:
         extra_nix_config: |
           sandbox = true
           max-jobs = 1
-    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/cachix-action@v15
-      if: needs.check_secrets.outputs.cachix == 'true'
-      with:
-        name: '${{ env.CACHIX_NAME }}'
-        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
-        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+    - uses: DeterminateSystems/magic-nix-cache-action@main
     # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user:
     # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces
     - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
       if: matrix.os == 'ubuntu-latest'
     - run: scripts/build-checks
-
-  # Steps to test CI automation in your own fork.
-  # Cachix:
-  # 1. Sign-up for https://www.cachix.org/
-  # 2. Create a cache for $githubuser-nix-install-tests
-  # 3. Create a cachix auth token and save it in https://github.com/$githubuser/nix/settings/secrets/actions in "Repository secrets" as CACHIX_AUTH_TOKEN
-  # Dockerhub:
-  # 1. Sign-up for https://hub.docker.com/
-  # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions)
-  # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork
-  check_secrets:
-    permissions:
-      contents: none
-    name: Check Cachix and Docker secrets present for installer tests
-    runs-on: ubuntu-latest
-    outputs:
-      cachix: ${{ steps.secret.outputs.cachix }}
-      docker: ${{ steps.secret.outputs.docker }}
-    steps:
-      - name: Check for secrets
-        id: secret
-        env:
-          _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }}
-          _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }}
-        run: |
-          echo "::set-output name=cachix::${{ env._CACHIX_SECRETS != '' }}"
-          echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}"
-
-  installer:
-    needs: [tests, check_secrets]
-    if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true'
-    runs-on: ubuntu-latest
-    outputs:
-      installerURL: ${{ steps.prepare-installer.outputs.installerURL }}
-    steps:
-    - uses: actions/checkout@v4
+    - run: scripts/prepare-installer-for-github-actions
+    - name: Upload installer tarball
+      uses: actions/upload-artifact@v4
       with:
-        fetch-depth: 0
-    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
-    - uses: cachix/install-nix-action@v30
-      with:
-        install_url: https://releases.nixos.org/nix/nix-2.20.3/install
-    - uses: cachix/cachix-action@v15
-      with:
-        name: '${{ env.CACHIX_NAME }}'
-        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
-        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-        cachixArgs: '-v'
-    - id: prepare-installer
-      run: scripts/prepare-installer-for-github-actions
+        name: installer-${{matrix.os}}
+        path: out/*
 
   installer_test:
-    needs: [installer, check_secrets]
-    if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true'
+    needs: [tests]
     strategy:
       fail-fast: false
       matrix:
@@ -107,11 +55,18 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v4
-    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
+    - name: Download installer tarball
+      uses: actions/download-artifact@v4
+      with:
+        name: installer-${{matrix.os}}
+        path: out
+    - name: Serving installer
+      id: serving_installer
+      run: ./scripts/serve-installer-for-github-actions
     - uses: cachix/install-nix-action@v30
       with:
-        install_url: '${{needs.installer.outputs.installerURL}}'
-        install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve"
+        install_url: 'http://localhost:8126/install'
+        install_options: "--tarball-url-prefix http://localhost:8126/"
     - run: sudo apt install fish zsh
       if: matrix.os == 'ubuntu-latest'
     - run: brew install fish
@@ -123,32 +78,50 @@ jobs:
     - run: exec bash -c "nix-channel --add https://releases.nixos.org/nixos/unstable/nixos-23.05pre466020.60c1d71f2ba nixpkgs"
     - run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello"
 
+  # Steps to test CI automation in your own fork.
+  # 1. Sign-up for https://hub.docker.com/
+  # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions)
+  # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork
+  check_secrets:
+    permissions:
+      contents: none
+    name: Check Docker secrets present for installer tests
+    runs-on: ubuntu-latest
+    outputs:
+      docker: ${{ steps.secret.outputs.docker }}
+    steps:
+      - name: Check for secrets
+        id: secret
+        env:
+          _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }}
+        run: |
+          echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}"
+
   docker_push_image:
-    needs: [check_secrets, tests, vm_tests]
+    needs: [tests, vm_tests, check_secrets]
     permissions:
       contents: read
       packages: write
     if: >-
+      needs.check_secrets.outputs.docker == 'true' &&
       github.event_name == 'push' &&
-      github.ref_name == 'master' &&
-      needs.check_secrets.outputs.cachix == 'true' &&
-      needs.check_secrets.outputs.docker == 'true'
+      github.ref_name == 'master'
     runs-on: ubuntu-latest
     steps:
+    - name: Check for secrets
+      id: secret
+      env:
+        _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }}
+      run: |
+        echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}"
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - uses: cachix/install-nix-action@v30
       with:
         install_url: https://releases.nixos.org/nix/nix-2.20.3/install
-    - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
+    - uses: DeterminateSystems/magic-nix-cache-action@main
     - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV
-    - uses: cachix/cachix-action@v15
-      if: needs.check_secrets.outputs.cachix == 'true'
-      with:
-        name: '${{ env.CACHIX_NAME }}'
-        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
-        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
     - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L
     - run: docker load -i ./result/image.tar.gz
     - run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION

flake.nix

@@ -186,7 +186,7 @@
       };
 
       checks = forAllSystems (system: {
-        binaryTarball = self.hydraJobs.binaryTarball.${system};
+        installerScriptForGHA = self.hydraJobs.installerScriptForGHA.${system};
         installTests = self.hydraJobs.installTests.${system};
         nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system};
         rl-next =
@@ -237,6 +237,8 @@
           inherit (nixpkgsFor.${system}.native)
             changelog-d;
           default = self.packages.${system}.nix;
+          installerScriptForGHA = self.hydraJobs.installerScriptForGHA.${system};
+          binaryTarball = self.hydraJobs.binaryTarball.${system};
           # TODO probably should be `nix-cli`
           nix = self.packages.${system}.nix-everything;
           nix-manual = nixpkgsFor.${system}.native.nixComponents.nix-manual;

packaging/hydra.nix

@@ -123,15 +123,10 @@ in
     self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
     self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu"
   ];
-  installerScriptForGHA = installScriptFor [
-    # Native
-    self.hydraJobs.binaryTarball."x86_64-linux"
-    self.hydraJobs.binaryTarball."aarch64-darwin"
-    # Cross
-    self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf"
-    self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
-    self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu"
-  ];
+
+  installerScriptForGHA = forAllSystems (system: nixpkgsFor.${system}.native.callPackage ../scripts/installer.nix {
+    tarballs = [ self.hydraJobs.binaryTarball.${system} ];
+  });
 
   # docker image with Nix inside
   dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage);

scripts/prepare-installer-for-github-actions

@@ -1,10 +1,11 @@
 #!/usr/bin/env bash
 
-set -e
+set -euo pipefail
 
-script=$(nix-build -A outputs.hydraJobs.installerScriptForGHA --no-out-link)
-installerHash=$(echo "$script" | cut -b12-43 -)
+nix build -L ".#installerScriptForGHA" ".#binaryTarball"
 
-installerURL=https://$CACHIX_NAME.cachix.org/serve/$installerHash/install
-
-echo "::set-output name=installerURL::$installerURL"
+mkdir -p out
+cp ./result/install "out/install"
+name="$(basename "$(realpath ./result-1)")"
+# everything before the first dash
+cp -r ./result-1 "out/${name%%-*}"

scripts/serve-installer-for-github-actions

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+if [[ ! -d out ]]; then
+  echo "run prepare-installer-for-github-actions first"
+  exit 1
+fi
+cd out
+PORT=${PORT:-8126}
+nohup python -m http.server "$PORT" >/dev/null 2>&1 &
+pid=$!
+
+while ! curl -s "http://localhost:$PORT"; do
+  sleep 1
+  if ! kill -0 $pid; then
+    echo "Failed to start http server"
+    exit 1
+  fi
+done
+
+echo 'To install nix, run the following command:'
+echo "sh <(curl http://localhost:$PORT/install) --tarball-url-prefix http://localhost:$PORT"