Merge remote-tracking branch 'origin/master' into lfs

tests/nixos/cgroups/default.nix

@@ -0,0 +1,40 @@
+{ nixpkgs, ... }:
+
+{
+  name = "cgroups";
+
+  nodes =
+    {
+      host =
+        { config, pkgs, ... }:
+        { virtualisation.additionalPaths = [ pkgs.stdenvNoCC ];
+          nix.extraOptions =
+            ''
+              extra-experimental-features = nix-command auto-allocate-uids cgroups
+              extra-system-features = uid-range
+            '';
+          nix.settings.use-cgroups = true;
+          nix.nixPath = [ "nixpkgs=${nixpkgs}" ];
+        };
+    };
+
+  testScript = { nodes }: ''
+    start_all()
+
+    host.wait_for_unit("multi-user.target")
+
+    # Start build in background
+    host.execute("NIX_REMOTE=daemon nix build --auto-allocate-uids --file ${./hang.nix} >&2 &")
+    service = "/sys/fs/cgroup/system.slice/nix-daemon.service"
+
+    # Wait for cgroups to be created
+    host.succeed(f"until [ -e {service}/nix-daemon ]; do sleep 1; done", timeout=30)
+    host.succeed(f"until [ -e {service}/nix-build-uid-* ]; do sleep 1; done", timeout=30)
+
+    # Check that there aren't processes where there shouldn't be, and that there are where there should be
+    host.succeed(f'[ -z "$(cat {service}/cgroup.procs)" ]')
+    host.succeed(f'[ -n "$(cat {service}/nix-daemon/cgroup.procs)" ]')
+    host.succeed(f'[ -n "$(cat {service}/nix-build-uid-*/cgroup.procs)" ]')
+  '';
+
+}

tests/nixos/cgroups/hang.nix

@@ -0,0 +1,10 @@
+{ }:
+
+with import <nixpkgs> {};
+
+runCommand "hang"
+  { requiredSystemFeatures = "uid-range";
+  }
+  ''
+    sleep infinity
+  ''

tests/nixos/chroot-store.nix

@@ -0,0 +1,31 @@
+{ lib, config, nixpkgs, ... }:
+
+let
+  pkgs = config.nodes.machine.nixpkgs.pkgs;
+  pkgA = pkgs.hello;
+  pkgB = pkgs.cowsay;
+in {
+  name = "chroot-store";
+
+  nodes =
+    { machine =
+        { config, lib, pkgs, ... }:
+        { virtualisation.writableStore = true;
+          virtualisation.additionalPaths = [ pkgA ];
+          environment.systemPackages = [ pkgB ];
+          nix.extraOptions = "experimental-features = nix-command";
+        };
+    };
+
+  testScript = { nodes }: ''
+    # fmt: off
+    start_all()
+
+    machine.succeed("nix copy --no-check-sigs --to /tmp/nix ${pkgA}")
+
+    machine.succeed("nix shell --store /tmp/nix ${pkgA} --command hello >&2")
+
+    # Test that /nix/store is available via an overlayfs mount.
+    machine.succeed("nix shell --store /tmp/nix ${pkgA} --command cowsay foo >&2")
+  '';
+}

tests/nixos/default.nix

@@ -4,20 +4,25 @@ let
 
   nixos-lib = import (nixpkgs + "/nixos/lib") { };
 
+  noTests = pkg: pkg.overrideAttrs (
+            finalAttrs: prevAttrs: {
+              doCheck = false;
+              doInstallCheck = false;
+            });
+
   # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests
   runNixOSTestFor = system: test:
     (nixos-lib.runTest {
       imports = [
         test
-
-        # Add the quickBuild attribute to the check packages
-        ./quick-build.nix
       ];
 
       hostPkgs = nixpkgsFor.${system}.native;
       defaults = {
         nixpkgs.pkgs = nixpkgsFor.${system}.native;
         nix.checkAllErrors = false;
+        # TODO: decide which packaging stage to use. `nix-cli` is efficient, but not the same as the user-facing `everything.nix` package (`default`). Perhaps a good compromise is `everything.nix` + `noTests` defined above?
+        nix.package = nixpkgsFor.${system}.native.nixComponents.nix-cli;
       };
       _module.args.nixpkgs = nixpkgs;
       _module.args.system = system;
@@ -29,7 +34,9 @@ let
       forNix = nixVersion: runNixOSTestFor system {
         imports = [test];
         defaults.nixpkgs.overlays = [(curr: prev: {
-          nix = (builtins.getFlake "nix/${nixVersion}").packages.${system}.nix;
+          nix = let
+            packages = (builtins.getFlake "nix/${nixVersion}").packages.${system};
+          in packages.nix-cli or packages.nix;
         })];
       };
     };
@@ -148,4 +155,12 @@ in
   user-sandboxing = runNixOSTestFor "x86_64-linux" ./user-sandboxing;
 
   s3-binary-cache-store = runNixOSTestFor "x86_64-linux" ./s3-binary-cache-store.nix;
+
+  fsync = runNixOSTestFor "x86_64-linux" ./fsync.nix;
+
+  cgroups = runNixOSTestFor "x86_64-linux" ./cgroups;
+
+  fetchurl = runNixOSTestFor "x86_64-linux" ./fetchurl.nix;
+
+  chrootStore = runNixOSTestFor "x86_64-linux" ./chroot-store.nix;
 }

tests/nixos/fetchurl.nix

@@ -0,0 +1,84 @@
+# Test whether builtin:fetchurl properly performs TLS certificate
+# checks on HTTPS servers.
+
+{ pkgs, ... }:
+
+let
+
+  makeTlsCert = name: pkgs.runCommand name {
+    nativeBuildInputs = with pkgs; [ openssl ];
+  } ''
+    mkdir -p $out
+    openssl req -x509 \
+      -subj '/CN=${name}/' -days 49710 \
+      -addext 'subjectAltName = DNS:${name}' \
+      -keyout "$out/key.pem" -newkey ed25519 \
+      -out "$out/cert.pem" -noenc
+  '';
+
+  goodCert = makeTlsCert "good";
+  badCert = makeTlsCert "bad";
+
+in
+
+{
+  name = "nss-preload";
+
+  nodes = {
+    machine = { pkgs, ... }: {
+      services.nginx = {
+        enable = true;
+
+        virtualHosts."good" = {
+          addSSL = true;
+          sslCertificate = "${goodCert}/cert.pem";
+          sslCertificateKey = "${goodCert}/key.pem";
+          root = pkgs.runCommand "nginx-root" {} ''
+            mkdir "$out"
+            echo 'hello world' > "$out/index.html"
+          '';
+        };
+
+        virtualHosts."bad" = {
+          addSSL = true;
+          sslCertificate = "${badCert}/cert.pem";
+          sslCertificateKey = "${badCert}/key.pem";
+          root = pkgs.runCommand "nginx-root" {} ''
+            mkdir "$out"
+            echo 'foobar' > "$out/index.html"
+          '';
+        };
+      };
+
+      security.pki.certificateFiles = [ "${goodCert}/cert.pem" ];
+
+      networking.hosts."127.0.0.1" = [ "good" "bad" ];
+
+      virtualisation.writableStore = true;
+
+      nix.settings.experimental-features = "nix-command";
+    };
+  };
+
+  testScript = ''
+    machine.wait_for_unit("nginx")
+    machine.wait_for_open_port(443)
+
+    out = machine.succeed("curl https://good/index.html")
+    assert out == "hello world\n"
+
+    out = machine.succeed("cat ${badCert}/cert.pem > /tmp/cafile.pem; curl --cacert /tmp/cafile.pem https://bad/index.html")
+    assert out == "foobar\n"
+
+    # Fetching from a server with a trusted cert should work.
+    machine.succeed("nix build --no-substitute --expr 'import <nix/fetchurl.nix> { url = \"https://good/index.html\"; hash = \"sha256-qUiQTy8PR5uPgZdpSzAYSw0u0cHNKh7A+4XSmaGSpEc=\"; }'")
+
+    # Fetching from a server with an untrusted cert should fail.
+    err = machine.fail("nix build --no-substitute --expr 'import <nix/fetchurl.nix> { url = \"https://bad/index.html\"; hash = \"sha256-rsBwZF/lPuOzdjBZN2E08FjMM3JHyXit0Xi2zN+wAZ8=\"; }' 2>&1")
+    print(err)
+    assert "SSL certificate problem: self-signed certificate" in err
+
+    # Fetching from a server with a trusted cert should work via environment variable override.
+    machine.succeed("NIX_SSL_CERT_FILE=/tmp/cafile.pem nix build --no-substitute --expr 'import <nix/fetchurl.nix> { url = \"https://bad/index.html\"; hash = \"sha256-rsBwZF/lPuOzdjBZN2E08FjMM3JHyXit0Xi2zN+wAZ8=\"; }'")
+  '';
+}

tests/nixos/fsync.nix

@@ -0,0 +1,39 @@
+{ lib, config, nixpkgs, pkgs, ... }:
+
+let
+  pkg1 = pkgs.go;
+in
+
+{
+  name = "fsync";
+
+  nodes.machine =
+    { config, lib, pkgs, ... }:
+    { virtualisation.emptyDiskImages = [ 1024 ];
+      environment.systemPackages = [ pkg1 ];
+      nix.settings.experimental-features = [ "nix-command" ];
+      nix.settings.fsync-store-paths = true;
+      nix.settings.require-sigs = false;
+      boot.supportedFilesystems = [ "ext4" "btrfs" "xfs" ];
+    };
+
+  testScript = { nodes }: ''
+    # fmt: off
+    for fs in ("ext4", "btrfs", "xfs"):
+      machine.succeed("mkfs.{} {} /dev/vdb".format(fs, "-F" if fs == "ext4" else "-f"))
+      machine.succeed("mkdir -p /mnt")
+      machine.succeed("mount /dev/vdb /mnt")
+      machine.succeed("sync")
+      machine.succeed("nix copy --offline ${pkg1} --to /mnt")
+      machine.crash()
+
+      machine.start()
+      machine.wait_for_unit("multi-user.target")
+      machine.succeed("mkdir -p /mnt")
+      machine.succeed("mount /dev/vdb /mnt")
+      machine.succeed("nix path-info --offline --store /mnt ${pkg1}")
+      machine.succeed("nix store verify --all --store /mnt --no-trust")
+
+      machine.succeed("umount /dev/vdb")
+  '';
+}

tests/nixos/functional/common.nix

@@ -24,43 +24,42 @@ in
     environment.systemPackages = let
       run-test-suite = pkgs.writeShellApplication {
         name = "run-test-suite";
-        runtimeInputs = [ pkgs.gnumake pkgs.jq pkgs.git ];
+        runtimeInputs = [
+          pkgs.meson
+          pkgs.ninja
+          pkgs.jq
+          pkgs.git
+
+          # Want to avoid `/run/current-system/sw/bin/bash` because we
+          # want a store path. Likewise for coreutils.
+          pkgs.bash
+          pkgs.coreutils
+        ];
         text = ''
           set -x
+
           cat /proc/sys/fs/file-max
           ulimit -Hn
           ulimit -Sn
-          cd ~
-          cp -r ${pkgs.nix.overrideAttrs (o: {
-            name = "nix-configured-source";
-            outputs = [ "out" ];
-            separateDebugInfo = false;
-            disallowedReferences = [ ];
-            buildPhase = ":";
-            checkPhase = ":";
-            installPhase = ''
-              cp -r . $out
-            '';
-            installCheckPhase = ":";
-            fixupPhase = ":";
-            doInstallCheck = true;
-          })} nix
-          chmod -R +w nix
-          cd nix
 
-          # Tests we don't need
-          echo >tests/functional/plugins/local.mk
-          sed -i tests/functional/local.mk \
-            -e 's!nix_tests += plugins\.sh!!' \
-            -e 's!nix_tests += test-libstoreconsumer\.sh!!' \
-            ;
+          cd ~
+
+          cp -r ${pkgs.nixComponents.nix-functional-tests.src} nix
+          chmod -R +w nix
+
+          chmod u+w nix/.version
+          echo ${pkgs.nixComponents.version} > nix/.version
 
           export isTestOnNixOS=1
-          export version=${config.nix.package.version}
+
           export NIX_REMOTE_=daemon
           export NIX_REMOTE=daemon
+
           export NIX_STORE=${builtins.storeDir}
-          make -j1 installcheck --keep-going
+
+          meson setup nix/tests/functional build
+          cd build
+          meson test -j1 --print-errorlogs
         '';
       };
     in [

tests/nixos/github-flakes.nix

@@ -181,8 +181,14 @@ in
     print(out)
     info = json.loads(out)
     assert info["revision"] == "${private-flake-rev}", f"revision mismatch: {info['revision']} != ${private-flake-rev}"
+    assert info["fingerprint"]
     cat_log()
 
+    # Fetching with the resolved URL should produce the same result.
+    info2 = json.loads(client.succeed(f"nix flake metadata {info['url']} --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0"))
+    print(info["fingerprint"], info2["fingerprint"])
+    assert info["fingerprint"] == info2["fingerprint"], "fingerprint mismatch"
+
     client.succeed("nix registry pin nixpkgs")
     client.succeed("nix flake metadata nixpkgs --tarball-ttl 0 >&2")
 

tests/nixos/quick-build.nix

@@ -1,47 +0,0 @@
-test@{ lib, extendModules, ... }:
-let
-  inherit (lib) mkOption types;
-in
-{
-  options = {
-    quickBuild = mkOption {
-      description = ''
-        Whether to perform a "quick" build of the Nix package to test.
-
-        When iterating on the functional tests, it's recommended to "set" this
-        to `true`, so that changes to the functional tests don't require any
-        recompilation of the package.
-        You can do so by building the `.quickBuild` attribute on the check package,
-        e.g:
-        ```console
-        nix build .#hydraJobs.functional_user.quickBuild
-        ```
-
-        We don't enable this by default to avoid the mostly unnecessary work of
-        performing an additional build of the package in cases where we build
-        the package normally anyway, such as in our pre-merge CI.
-      '';
-      type = types.bool;
-      default = false;
-    };
-  };
-
-  config = {
-    passthru.quickBuild = 
-      let withQuickBuild = extendModules { modules = [{ quickBuild = true; }]; };
-      in withQuickBuild.config.test;
-
-    defaults = { pkgs, ... }: {
-      config = lib.mkIf test.config.quickBuild {
-        nix.package = pkgs.nix_noTests;
-
-        system.forbiddenDependenciesRegexes = [
-          # This would indicate that the quickBuild feature is broken.
-          # It could happen if NixOS has a dependency on pkgs.nix instead of
-          # config.nix.package somewhere.
-          (builtins.unsafeDiscardStringContext pkgs.nix.outPath)
-        ];
-      };
-    };
-  };
-}

tests/nixos/remote-builds.nix

@@ -81,6 +81,17 @@ in
             virtualisation.additionalPaths = [ config.system.build.extraUtils ];
             nix.settings.substituters = lib.mkForce [ ];
             programs.ssh.extraConfig = "ConnectTimeout 30";
+            environment.systemPackages = [
+              # `bad-shell` is used to make sure Nix works an environment with a misbehaving shell.
+              #
+              # More realistically, a bad shell would still run the command ("echo started")
+              # but considering that our solution is to avoid this shell (set via $SHELL), we
+              # don't need to bother with a more functional mock shell.
+              (pkgs.writeScriptBin "bad-shell" ''
+                #!${pkgs.runtimeShell}
+                echo "Hello, I am a broken shell"
+              '')
+            ];
           };
       };
 
@@ -114,9 +125,13 @@ in
             'echo hello world on $(hostname)' >&2
         """)
 
+      # Check that SSH uses SHELL for LocalCommand, as expected, and check that
+      # our test setup here is working. The next test will use this bad SHELL.
+      client.succeed(f"SHELL=$(which bad-shell) ssh -oLocalCommand='true' -oPermitLocalCommand=yes {builder1.name} 'echo hello world' | grep -F 'Hello, I am a broken shell'")
+
       # Perform a build and check that it was performed on the builder.
       out = client.succeed(
-        "nix-build ${expr nodes.client 1} 2> build-output",
+        "SHELL=$(which bad-shell) nix-build ${expr nodes.client 1} 2> build-output",
         "grep -q Hello build-output"
       )
       builder1.succeed(f"test -e {out}")

tests/nixos/s3-binary-cache-store.nix

@@ -12,7 +12,7 @@ let
   storeUrl = "s3://my-cache?endpoint=http://server:9000&region=eu-west-1";
 
 in {
-  name = "nix-copy-closure";
+  name = "s3-binary-cache-store";
 
   nodes =
     { server =
@@ -51,6 +51,9 @@ in {
 
     server.succeed("${env} nix copy --to '${storeUrl}' ${pkgA}")
 
+    # Test fetchurl on s3:// URLs while we're at it.
+    client.succeed("${env} nix eval --impure --expr 'builtins.fetchurl { name = \"foo\"; url = \"s3://my-cache/nix-cache-info?endpoint=http://server:9000&region=eu-west-1\"; }'")
+
     # Copy a package from the binary cache.
     client.fail("nix path-info ${pkgA}")