diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb70fae87..29cb33f56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,7 @@ jobs: extra_nix_config: | sandbox = true max-jobs = 1 + - uses: DeterminateSystems/magic-nix-cache-action@main # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 @@ -133,6 +134,7 @@ jobs: - uses: cachix/install-nix-action@v31 with: install_url: https://releases.nixos.org/nix/nix-2.20.3/install + - uses: DeterminateSystems/magic-nix-cache-action@main - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L - run: docker load -i ./result/image.tar.gz @@ -174,6 +176,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@main + - uses: DeterminateSystems/magic-nix-cache-action@main - run: | nix build -L \ .#hydraJobs.tests.functional_user \ @@ -199,4 +202,5 @@ jobs: repository: NixOS/flake-regressions-data path: flake-regressions/tests - uses: DeterminateSystems/nix-installer-action@main + - uses: DeterminateSystems/magic-nix-cache-action@main - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh diff --git a/doc/manual/source/language/advanced-attributes.md b/doc/manual/source/language/advanced-attributes.md index a939847e1..34c3b636b 100644 --- a/doc/manual/source/language/advanced-attributes.md +++ b/doc/manual/source/language/advanced-attributes.md @@ -53,23 +53,13 @@ Derivations can declare some infrequently used optional attributes. - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ If the special attribute `__structuredAttrs` is set to `true`, the other derivation - attributes are serialised into a file in JSON format. The environment variable - `NIX_ATTRS_JSON_FILE` points to the exact location of that file both in a build - and a [`nix-shell`](../command-ref/nix-shell.md). This obviates the need for - [`passAsFile`](#adv-attr-passAsFile) since JSON files have no size restrictions, - unlike process environments. + attributes are serialised into a file in JSON format. - It also makes it possible to tweak derivation settings in a structured way; see - [`outputChecks`](#adv-attr-outputChecks) for example. + This obviates the need for [`passAsFile`](#adv-attr-passAsFile) since JSON files have no size restrictions, unlike process environments. + It also makes it possible to tweak derivation settings in a structured way; + see [`outputChecks`](#adv-attr-outputChecks) for example. - As a convenience to Bash builders, - Nix writes a script that initialises shell variables - corresponding to all attributes that are representable in Bash. The - environment variable `NIX_ATTRS_SH_FILE` points to the exact - location of the script, both in a build and a - [`nix-shell`](../command-ref/nix-shell.md). This includes non-nested - (associative) arrays. For example, the attribute `hardening.format = true` - ends up as the Bash associative array element `${hardening[format]}`. + See the [corresponding section in the derivation page](@docroot@/store/derivation/index.md#structured-attrs) for further details. > **Warning** > diff --git a/doc/manual/source/protocols/json/derivation.md b/doc/manual/source/protocols/json/derivation.md index 205b75fa5..04881776a 100644 --- a/doc/manual/source/protocols/json/derivation.md +++ b/doc/manual/source/protocols/json/derivation.md @@ -91,3 +91,7 @@ is a JSON object with the following fields: * `env`: The environment passed to the `builder`. + +* `structuredAttrs`: + [Strucutured Attributes](@docroot@/store/derivation/index.md#structured-attrs), only defined if the derivation contains them. + Structured attributes are JSON, and thus embedded as-is. diff --git a/doc/manual/source/store/derivation/index.md b/doc/manual/source/store/derivation/index.md index 16ffc0923..147330df6 100644 --- a/doc/manual/source/store/derivation/index.md +++ b/doc/manual/source/store/derivation/index.md @@ -138,6 +138,17 @@ See [Wikipedia](https://en.wikipedia.org/wiki/Argv) for details. Environment variables which will be passed to the [builder](#builder) executable. +#### Structured Attributes {#structured-attrs} + +Nix also has special support for embedding JSON in the derivations. + +The environment variable `NIX_ATTRS_JSON_FILE` points to the exact location of that file both in a build and a [`nix-shell`](@docroot@/command-ref/nix-shell.md). + +As a convenience to Bash builders, Nix writes a script that initialises shell variables corresponding to all attributes that are representable in Bash. +The environment variable `NIX_ATTRS_SH_FILE` points to the exact location of the script, both in a build and a [`nix-shell`](@docroot@/command-ref/nix-shell.md). +This includes non-nested (associative) arrays. +For example, the attribute `hardening.format = true` ends up as the Bash associative array element `${hardening[format]}`. + ### Placeholders Placeholders are opaque values used within the [process creation fields] to [store objects] for which we don't yet know [store path]s. diff --git a/doc/manual/source/store/store-object.md b/doc/manual/source/store/store-object.md index 115e107fb..10c2384fa 100644 --- a/doc/manual/source/store/store-object.md +++ b/doc/manual/source/store/store-object.md @@ -18,14 +18,14 @@ In particular, the edge corresponding to a reference is from the store object th References other than a self-reference must not form a cycle. The graph of references excluding self-references thus forms a [directed acyclic graph]. -[directed acyclic graph]: @docroot@/glossary.md#gloss-directed acyclic graph +[directed acyclic graph]: @docroot@/glossary.md#gloss-directed-acyclic-graph We can take the [transitive closure] of the references graph, which any pair of store objects have an edge not if there is a single reference from the first to the second, but a path of one or more references from the first to the second. The *requisites* of a store object are all store objects reachable by paths of references which start with given store object's references. [transitive closure]: https://en.wikipedia.org/wiki/Transitive_closure -We can also take the [transpose graph] ofthe references graph, where we reverse the orientation of all edges. +We can also take the [transpose graph] of the references graph, where we reverse the orientation of all edges. The *referrers* of a store object are the store objects that reference it. [transpose graph]: https://en.wikipedia.org/wiki/Transpose_graph diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json index 7d3c932b2..183148b29 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json @@ -5,7 +5,6 @@ ], "builder": "/bin/bash", "env": { - "__json": "{\"builder\":\"/bin/bash\",\"name\":\"advanced-attributes-structured-attrs-defaults\",\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"dev\"],\"system\":\"my-system\"}", "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz", "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9" }, @@ -22,5 +21,16 @@ "method": "nar" } }, + "structuredAttrs": { + "builder": "/bin/bash", + "name": "advanced-attributes-structured-attrs-defaults", + "outputHashAlgo": "sha256", + "outputHashMode": "recursive", + "outputs": [ + "out", + "dev" + ], + "system": "my-system" + }, "system": "my-system" } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json index a421efea7..ec044d778 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json @@ -5,7 +5,6 @@ ], "builder": "/bin/bash", "env": { - "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"refs2\":[\"/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g\"],\"disallowedRequisites\":[\"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"allowedRequisites\":[\"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", "bin": "/04f3da1kmbr67m3gzxikmsl4vjz5zf777sv6m14ahv22r65aac9m", "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz", "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9" @@ -44,5 +43,62 @@ "method": "nar" } }, + "structuredAttrs": { + "__darwinAllowLocalNetworking": true, + "__impureHostDeps": [ + "/usr/bin/ditto" + ], + "__noChroot": true, + "__sandboxProfile": "sandcastle", + "allowSubstitutes": false, + "builder": "/bin/bash", + "exportReferencesGraph": { + "refs1": [ + "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9" + ], + "refs2": [ + "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" + ] + }, + "impureEnvVars": [ + "UNICORN" + ], + "name": "advanced-attributes-structured-attrs", + "outputChecks": { + "bin": { + "disallowedReferences": [ + "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g" + ], + "disallowedRequisites": [ + "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8" + ] + }, + "dev": { + "maxClosureSize": 5909, + "maxSize": 789 + }, + "out": { + "allowedReferences": [ + "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9" + ], + "allowedRequisites": [ + "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z" + ] + } + }, + "outputHashAlgo": "sha256", + "outputHashMode": "recursive", + "outputs": [ + "out", + "bin", + "dev" + ], + "preferLocalBuild": true, + "requiredSystemFeatures": [ + "rainbow", + "uid-range" + ], + "system": "my-system" + }, "system": "my-system" } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json index 473d006ac..f5349e6c3 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json @@ -5,7 +5,6 @@ ], "builder": "/bin/bash", "env": { - "__json": "{\"builder\":\"/bin/bash\",\"name\":\"advanced-attributes-structured-attrs-defaults\",\"outputs\":[\"out\",\"dev\"],\"system\":\"my-system\"}", "dev": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev", "out": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" }, @@ -20,5 +19,14 @@ "path": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" } }, + "structuredAttrs": { + "builder": "/bin/bash", + "name": "advanced-attributes-structured-attrs-defaults", + "outputs": [ + "out", + "dev" + ], + "system": "my-system" + }, "system": "my-system" } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json index d68502d56..b8d566462 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json @@ -5,7 +5,6 @@ ], "builder": "/bin/bash", "env": { - "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"refs2\":[\"/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar\"],\"disallowedRequisites\":[\"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"allowedRequisites\":[\"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", "bin": "/nix/store/33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin", "dev": "/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev", "out": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs" @@ -41,5 +40,60 @@ "path": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs" } }, + "structuredAttrs": { + "__darwinAllowLocalNetworking": true, + "__impureHostDeps": [ + "/usr/bin/ditto" + ], + "__noChroot": true, + "__sandboxProfile": "sandcastle", + "allowSubstitutes": false, + "builder": "/bin/bash", + "exportReferencesGraph": { + "refs1": [ + "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo" + ], + "refs2": [ + "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" + ] + }, + "impureEnvVars": [ + "UNICORN" + ], + "name": "advanced-attributes-structured-attrs", + "outputChecks": { + "bin": { + "disallowedReferences": [ + "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar" + ], + "disallowedRequisites": [ + "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev" + ] + }, + "dev": { + "maxClosureSize": 5909, + "maxSize": 789 + }, + "out": { + "allowedReferences": [ + "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo" + ], + "allowedRequisites": [ + "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev" + ] + } + }, + "outputs": [ + "out", + "bin", + "dev" + ], + "preferLocalBuild": true, + "requiredSystemFeatures": [ + "rainbow", + "uid-range" + ], + "system": "my-system" + }, "system": "my-system" } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 7c13b4f63..0657a7499 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1333,6 +1333,11 @@ nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const res["args"] = args; res["env"] = env; + if (auto it = env.find("__json"); it != env.end()) { + res["env"].erase("__json"); + res["structuredAttrs"] = nlohmann::json::parse(it->second); + } + return res; } @@ -1396,7 +1401,17 @@ Derivation Derivation::fromJSON( res.platform = getString(valueAt(json, "system")); res.builder = getString(valueAt(json, "builder")); res.args = getStringList(valueAt(json, "args")); - res.env = getStringMap(valueAt(json, "env")); + + auto envJson = valueAt(json, "env"); + try { + res.env = getStringMap(envJson); + } catch (Error & e) { + e.addTrace({}, "while reading key 'env'"); + throw; + } + + if (auto structuredAttrs = get(json, "structuredAttrs")) + res.env.insert_or_assign("__json", structuredAttrs->dump()); return res; } diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 1f15cc9e9..85b586373 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -700,7 +700,7 @@ void DerivationBuilderImpl::startBuilder() // since aarch64-darwin has Rosetta 2, this user can actually run x86_64-darwin on their hardware - we should tell them to run the command to install Darwin 2 if (drv.platform == "x86_64-darwin" && settings.thisSystem == "aarch64-darwin") - msg += fmt("\nNote: run `%s` to run programs for x86_64-darwin", Magenta("/usr/sbin/softwareupdate --install-rosetta")); + msg += fmt("\nNote: run `%s` to run programs for x86_64-darwin", Magenta("/usr/sbin/softwareupdate --install-rosetta && launchctl stop org.nixos.nix-daemon")); throw BuildError(msg); } diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index 465676b41..2bd9b4aaf 100755 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -50,4 +50,4 @@ expectStderr 0 nix-instantiate --expr "$hackyExpr" --eval --strict | grepQuiet " # Check it works with the expected structured attrs hacky=$(nix-instantiate --expr "$hackyExpr") -nix derivation show "$hacky" | jq --exit-status '."'"$hacky"'".env.__json | fromjson | . == {"a": 1}' +nix derivation show "$hacky" | jq --exit-status '."'"$hacky"'".structuredAttrs | . == {"a": 1}'