Merge pull request #13349 from obsidiansystems/structured-attrs-json

Introduce top-level `structuredAttrs` field in JSON derivation format

doc/manual/source/language/advanced-attributes.md

@@ -53,23 +53,13 @@ Derivations can declare some infrequently used optional attributes.
 
   - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\
     If the special attribute `__structuredAttrs` is set to `true`, the other derivation
-    attributes are serialised into a file in JSON format. The environment variable
+    attributes are serialised into a file in JSON format.
-    `NIX_ATTRS_JSON_FILE` points to the exact location of that file both in a build
-    and a [`nix-shell`](../command-ref/nix-shell.md). This obviates the need for
-    [`passAsFile`](#adv-attr-passAsFile) since JSON files have no size restrictions,
-    unlike process environments.
 
-    It also makes it possible to tweak derivation settings in a structured way; see
+    This obviates the need for [`passAsFile`](#adv-attr-passAsFile) since JSON files have no size restrictions, unlike process environments.
-    [`outputChecks`](#adv-attr-outputChecks) for example.
+    It also makes it possible to tweak derivation settings in a structured way;
+    see [`outputChecks`](#adv-attr-outputChecks) for example.
 
-    As a convenience to Bash builders,
+    See the [corresponding section in the derivation page](@docroot@/store/derivation/index.md#structured-attrs) for further details.
-    Nix writes a script that initialises shell variables
-    corresponding to all attributes that are representable in Bash. The
-    environment variable `NIX_ATTRS_SH_FILE` points to the exact
-    location of the script, both in a build and a
-    [`nix-shell`](../command-ref/nix-shell.md). This includes non-nested
-    (associative) arrays. For example, the attribute `hardening.format = true`
-    ends up as the Bash associative array element `${hardening[format]}`.
 
     > **Warning**
     >

doc/manual/source/protocols/json/derivation.md

@@ -91,3 +91,7 @@ is a JSON object with the following fields:
 
 * `env`:
   The environment passed to the `builder`.
+
+* `structuredAttrs`:
+  [Strucutured Attributes](@docroot@/store/derivation/index.md#structured-attrs), only defined if the derivation contains them.
+  Structured attributes are JSON, and thus embedded as-is.

doc/manual/source/store/derivation/index.md

@@ -138,6 +138,17 @@ See [Wikipedia](https://en.wikipedia.org/wiki/Argv) for details.
 
 Environment variables which will be passed to the [builder](#builder) executable.
 
+#### Structured Attributes {#structured-attrs}
+
+Nix also has special support for embedding JSON in the derivations.
+
+The environment variable `NIX_ATTRS_JSON_FILE` points to the exact location of that file both in a build and a [`nix-shell`](@docroot@/command-ref/nix-shell.md).
+
+As a convenience to Bash builders, Nix writes a script that initialises shell variables corresponding to all attributes that are representable in Bash.
+The environment variable `NIX_ATTRS_SH_FILE` points to the exact location of the script, both in a build and a [`nix-shell`](@docroot@/command-ref/nix-shell.md).
+This includes non-nested (associative) arrays.
+For example, the attribute `hardening.format = true` ends up as the Bash associative array element `${hardening[format]}`.
+
 ### Placeholders
 
 Placeholders are opaque values used within the [process creation fields] to [store objects] for which we don't yet know [store path]s.

src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json

@@ -5,7 +5,6 @@
   ],
   "builder": "/bin/bash",
   "env": {
-    "__json": "{\"builder\":\"/bin/bash\",\"name\":\"advanced-attributes-structured-attrs-defaults\",\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"dev\"],\"system\":\"my-system\"}",
     "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz",
     "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"
   },
@@ -22,5 +21,16 @@
       "method": "nar"
     }
   },
+  "structuredAttrs": {
+    "builder": "/bin/bash",
+    "name": "advanced-attributes-structured-attrs-defaults",
+    "outputHashAlgo": "sha256",
+    "outputHashMode": "recursive",
+    "outputs": [
+      "out",
+      "dev"
+    ],
+    "system": "my-system"
+  },
   "system": "my-system"
 }

src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json

@@ -5,7 +5,6 @@
   ],
   "builder": "/bin/bash",
   "env": {
-    "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"refs2\":[\"/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g\"],\"disallowedRequisites\":[\"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"allowedRequisites\":[\"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}",
     "bin": "/04f3da1kmbr67m3gzxikmsl4vjz5zf777sv6m14ahv22r65aac9m",
     "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz",
     "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"
@@ -44,5 +43,62 @@
       "method": "nar"
     }
   },
+  "structuredAttrs": {
+    "__darwinAllowLocalNetworking": true,
+    "__impureHostDeps": [
+      "/usr/bin/ditto"
+    ],
+    "__noChroot": true,
+    "__sandboxProfile": "sandcastle",
+    "allowSubstitutes": false,
+    "builder": "/bin/bash",
+    "exportReferencesGraph": {
+      "refs1": [
+        "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
+      ],
+      "refs2": [
+        "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv"
+      ]
+    },
+    "impureEnvVars": [
+      "UNICORN"
+    ],
+    "name": "advanced-attributes-structured-attrs",
+    "outputChecks": {
+      "bin": {
+        "disallowedReferences": [
+          "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"
+        ],
+        "disallowedRequisites": [
+          "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"
+        ]
+      },
+      "dev": {
+        "maxClosureSize": 5909,
+        "maxSize": 789
+      },
+      "out": {
+        "allowedReferences": [
+          "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
+        ],
+        "allowedRequisites": [
+          "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"
+        ]
+      }
+    },
+    "outputHashAlgo": "sha256",
+    "outputHashMode": "recursive",
+    "outputs": [
+      "out",
+      "bin",
+      "dev"
+    ],
+    "preferLocalBuild": true,
+    "requiredSystemFeatures": [
+      "rainbow",
+      "uid-range"
+    ],
+    "system": "my-system"
+  },
   "system": "my-system"
 }

src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json

@@ -5,7 +5,6 @@
   ],
   "builder": "/bin/bash",
   "env": {
-    "__json": "{\"builder\":\"/bin/bash\",\"name\":\"advanced-attributes-structured-attrs-defaults\",\"outputs\":[\"out\",\"dev\"],\"system\":\"my-system\"}",
     "dev": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev",
     "out": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults"
   },
@@ -20,5 +19,14 @@
       "path": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults"
     }
   },
+  "structuredAttrs": {
+    "builder": "/bin/bash",
+    "name": "advanced-attributes-structured-attrs-defaults",
+    "outputs": [
+      "out",
+      "dev"
+    ],
+    "system": "my-system"
+  },
   "system": "my-system"
 }

src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json

@@ -5,7 +5,6 @@
   ],
   "builder": "/bin/bash",
   "env": {
-    "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"exportReferencesGraph\":{\"refs1\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"refs2\":[\"/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv\"]},\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar\"],\"disallowedRequisites\":[\"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"allowedRequisites\":[\"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}",
     "bin": "/nix/store/33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin",
     "dev": "/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev",
     "out": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs"
@@ -41,5 +40,60 @@
       "path": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs"
     }
   },
+  "structuredAttrs": {
+    "__darwinAllowLocalNetworking": true,
+    "__impureHostDeps": [
+      "/usr/bin/ditto"
+    ],
+    "__noChroot": true,
+    "__sandboxProfile": "sandcastle",
+    "allowSubstitutes": false,
+    "builder": "/bin/bash",
+    "exportReferencesGraph": {
+      "refs1": [
+        "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
+      ],
+      "refs2": [
+        "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv"
+      ]
+    },
+    "impureEnvVars": [
+      "UNICORN"
+    ],
+    "name": "advanced-attributes-structured-attrs",
+    "outputChecks": {
+      "bin": {
+        "disallowedReferences": [
+          "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"
+        ],
+        "disallowedRequisites": [
+          "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
+        ]
+      },
+      "dev": {
+        "maxClosureSize": 5909,
+        "maxSize": 789
+      },
+      "out": {
+        "allowedReferences": [
+          "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
+        ],
+        "allowedRequisites": [
+          "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"
+        ]
+      }
+    },
+    "outputs": [
+      "out",
+      "bin",
+      "dev"
+    ],
+    "preferLocalBuild": true,
+    "requiredSystemFeatures": [
+      "rainbow",
+      "uid-range"
+    ],
+    "system": "my-system"
+  },
   "system": "my-system"
 }

src/libstore/derivations.cc

@@ -1333,6 +1333,11 @@ nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const
     res["args"] = args;
     res["env"] = env;
 
+    if (auto it = env.find("__json"); it != env.end()) {
+        res["env"].erase("__json");
+        res["structuredAttrs"] = nlohmann::json::parse(it->second);
+    }
+
     return res;
 }
 
@@ -1396,7 +1401,17 @@ Derivation Derivation::fromJSON(
     res.platform = getString(valueAt(json, "system"));
     res.builder = getString(valueAt(json, "builder"));
     res.args = getStringList(valueAt(json, "args"));
-    res.env = getStringMap(valueAt(json, "env"));
+
+    auto envJson = valueAt(json, "env");
+    try {
+        res.env = getStringMap(envJson);
+    } catch (Error & e) {
+        e.addTrace({}, "while reading key 'env'");
+        throw;
+    }
+
+    if (auto structuredAttrs = get(json, "structuredAttrs"))
+        res.env.insert_or_assign("__json", structuredAttrs->dump());
 
     return res;
 }

tests/functional/structured-attrs.sh

@@ -50,4 +50,4 @@ expectStderr 0 nix-instantiate --expr "$hackyExpr" --eval --strict | grepQuiet "
 
 # Check it works with the expected structured attrs
 hacky=$(nix-instantiate --expr "$hackyExpr")
-nix derivation show "$hacky" | jq --exit-status '."'"$hacky"'".env.__json | fromjson | . == {"a": 1}'
+nix derivation show "$hacky" | jq --exit-status '."'"$hacky"'".structuredAttrs | . == {"a": 1}'