wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-25 10:41:16 +02:00
Code Activity

Merge pull request #12410 from obsidiansystems/derivation-options-2

Browse source 
Scrap `ParsedDerivation` for parts
This commit is contained in:
Jörg Thalheim 2025-04-15 18:36:11 +02:00 • committed by GitHub
commit cf6da9234c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 223 additions and 216 deletions
Download patch file Download diff file Expand all files Collapse all files

56
src/libstore-tests/derivation-advanced-attrs.cc
View file

@ -108,10 +108,10 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults)
auto drvPath = writeDerivation(*this->store, got, NoRepair, true); auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
ParsedDerivation parsedDrv(got); auto parsedDrv = StructuredAttrs::tryParse(got.env);
DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
EXPECT_TRUE(!parsedDrv.hasStructuredAttrs()); EXPECT_TRUE(!parsedDrv);
EXPECT_EQ(options.additionalSandboxProfile, ""); EXPECT_EQ(options.additionalSandboxProfile, "");
EXPECT_EQ(options.noChroot, false); EXPECT_EQ(options.noChroot, false);
@ -143,8 +143,8 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_defaults)
auto drvPath = writeDerivation(*this->store, got, NoRepair, true); auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
ParsedDerivation parsedDrv(got); auto parsedDrv = StructuredAttrs::tryParse(got.env);
DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{}); EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{});
}); });
@ -157,8 +157,8 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_defaults)
auto drvPath = writeDerivation(*this->store, got, NoRepair, true); auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
ParsedDerivation parsedDrv(got); auto parsedDrv = StructuredAttrs::tryParse(got.env);
DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"}); EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"});
}); });
@ -171,10 +171,10 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes)
auto drvPath = writeDerivation(*this->store, got, NoRepair, true); auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
ParsedDerivation parsedDrv(got); auto parsedDrv = StructuredAttrs::tryParse(got.env);
DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
EXPECT_TRUE(!parsedDrv.hasStructuredAttrs()); EXPECT_TRUE(!parsedDrv);
EXPECT_EQ(options.additionalSandboxProfile, "sandcastle"); EXPECT_EQ(options.additionalSandboxProfile, "sandcastle");
EXPECT_EQ(options.noChroot, true); EXPECT_EQ(options.noChroot, true);
@ -195,8 +195,8 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes)
auto drvPath = writeDerivation(*this->store, got, NoRepair, true); auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
ParsedDerivation parsedDrv(got); auto parsedDrv = StructuredAttrs::tryParse(got.env);
DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
EXPECT_EQ( EXPECT_EQ(
options.exportReferencesGraph, options.exportReferencesGraph,
@ -243,8 +243,8 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes)
auto drvPath = writeDerivation(*this->store, got, NoRepair, true); auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
ParsedDerivation parsedDrv(got); auto parsedDrv = StructuredAttrs::tryParse(got.env);
DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
EXPECT_EQ( EXPECT_EQ(
options.exportReferencesGraph, options.exportReferencesGraph,
@ -296,10 +296,10 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_d
auto drvPath = writeDerivation(*this->store, got, NoRepair, true); auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
ParsedDerivation parsedDrv(got); auto parsedDrv = StructuredAttrs::tryParse(got.env);
DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
EXPECT_TRUE(parsedDrv.hasStructuredAttrs()); EXPECT_TRUE(parsedDrv);
EXPECT_EQ(options.additionalSandboxProfile, ""); EXPECT_EQ(options.additionalSandboxProfile, "");
EXPECT_EQ(options.noChroot, false); EXPECT_EQ(options.noChroot, false);
@ -330,8 +330,8 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_defaults)
auto drvPath = writeDerivation(*this->store, got, NoRepair, true); auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
ParsedDerivation parsedDrv(got); auto parsedDrv = StructuredAttrs::tryParse(got.env);
DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{}); EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{});
}); });
@ -344,8 +344,8 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_default
auto drvPath = writeDerivation(*this->store, got, NoRepair, true); auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
ParsedDerivation parsedDrv(got); auto parsedDrv = StructuredAttrs::tryParse(got.env);
DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"}); EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"});
}); });
@ -358,10 +358,10 @@ TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs)
auto drvPath = writeDerivation(*this->store, got, NoRepair, true); auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
ParsedDerivation parsedDrv(got); auto parsedDrv = StructuredAttrs::tryParse(got.env);
DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
EXPECT_TRUE(parsedDrv.hasStructuredAttrs()); EXPECT_TRUE(parsedDrv);
EXPECT_EQ(options.additionalSandboxProfile, "sandcastle"); EXPECT_EQ(options.additionalSandboxProfile, "sandcastle");
EXPECT_EQ(options.noChroot, true); EXPECT_EQ(options.noChroot, true);
@ -392,8 +392,8 @@ TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs)
auto drvPath = writeDerivation(*this->store, got, NoRepair, true); auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
ParsedDerivation parsedDrv(got); auto parsedDrv = StructuredAttrs::tryParse(got.env);
DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
EXPECT_EQ( EXPECT_EQ(
options.exportReferencesGraph, options.exportReferencesGraph,
@ -445,8 +445,8 @@ TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs)
auto drvPath = writeDerivation(*this->store, got, NoRepair, true); auto drvPath = writeDerivation(*this->store, got, NoRepair, true);
ParsedDerivation parsedDrv(got); auto parsedDrv = StructuredAttrs::tryParse(got.env);
DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); DerivationOptions options = DerivationOptions::fromStructuredAttrs(got.env, parsedDrv ? &*parsedDrv : nullptr);
EXPECT_EQ( EXPECT_EQ(
options.exportReferencesGraph, options.exportReferencesGraph,

7
src/libstore/build/derivation-goal.cc
View file

@ -180,9 +180,12 @@ Goal::Co DerivationGoal::haveDerivation()
{ {
trace("have derivation"); trace("have derivation");
parsedDrv = std::make_unique<ParsedDerivation>(*drv); if (auto parsedOpt = StructuredAttrs::tryParse(drv->env)) {
parsedDrv = std::make_unique<StructuredAttrs>(*parsedOpt);
}
try { try {
drvOptions = std::make_unique<DerivationOptions>(DerivationOptions::fromParsedDerivation(*parsedDrv)); drvOptions = std::make_unique<DerivationOptions>(
DerivationOptions::fromStructuredAttrs(drv->env, parsedDrv.get()));
} catch (Error & e) { } catch (Error & e) {
e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath)); e.addTrace({}, "while parsing derivation '%s'", worker.store.printStorePath(drvPath));
throw; throw;

140
src/libstore/derivation-options.cc
View file

@ -1,6 +1,8 @@
#include "nix/store/derivation-options.hh" #include "nix/store/derivation-options.hh"
#include "nix/util/json-utils.hh" #include "nix/util/json-utils.hh"
#include "nix/store/parsed-derivations.hh" #include "nix/store/parsed-derivations.hh"
#include "nix/store/derivations.hh"
#include "nix/store/store-api.hh"
#include "nix/util/types.hh" #include "nix/util/types.hh"
#include "nix/util/util.hh" #include "nix/util/util.hh"
@ -11,38 +13,112 @@
namespace nix { namespace nix {
static std::optional<std::string>
getStringAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name)
{
if (parsed) {
auto i = parsed->structuredAttrs.find(name);
if (i == parsed->structuredAttrs.end())
return {};
else {
if (!i->is_string())
throw Error("attribute '%s' of must be a string", name);
return i->get<std::string>();
}
} else {
auto i = env.find(name);
if (i == env.end())
return {};
else
return i->second;
}
}
static bool getBoolAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name, bool def)
{
if (parsed) {
auto i = parsed->structuredAttrs.find(name);
if (i == parsed->structuredAttrs.end())
return def;
else {
if (!i->is_boolean())
throw Error("attribute '%s' must be a Boolean", name);
return i->get<bool>();
}
} else {
auto i = env.find(name);
if (i == env.end())
return def;
else
return i->second == "1";
}
}
static std::optional<Strings>
getStringsAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name)
{
if (parsed) {
auto i = parsed->structuredAttrs.find(name);
if (i == parsed->structuredAttrs.end())
return {};
else {
if (!i->is_array())
throw Error("attribute '%s' must be a list of strings", name);
Strings res;
for (auto j = i->begin(); j != i->end(); ++j) {
if (!j->is_string())
throw Error("attribute '%s' must be a list of strings", name);
res.push_back(j->get<std::string>());
}
return res;
}
} else {
auto i = env.find(name);
if (i == env.end())
return {};
else
return tokenizeString<Strings>(i->second);
}
}
static std::optional<StringSet>
getStringSetAttr(const StringMap & env, const StructuredAttrs * parsed, const std::string & name)
{
auto ss = getStringsAttr(env, parsed, name);
return ss ? (std::optional{StringSet{ss->begin(), ss->end()}}) : (std::optional<StringSet>{});
}
using OutputChecks = DerivationOptions::OutputChecks; using OutputChecks = DerivationOptions::OutputChecks;
using OutputChecksVariant = std::variant<OutputChecks, std::map<std::string, OutputChecks>>; using OutputChecksVariant = std::variant<OutputChecks, std::map<std::string, OutputChecks>>;
DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation & parsed, bool shouldWarn) DerivationOptions
DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn)
{ {
DerivationOptions defaults = {}; DerivationOptions defaults = {};
auto structuredAttrs = parsed.structuredAttrs.get(); if (shouldWarn && parsed) {
if (get(parsed->structuredAttrs, "allowedReferences")) {
if (shouldWarn && structuredAttrs) {
if (get(*structuredAttrs, "allowedReferences")) {
warn( warn(
"'structuredAttrs' disables the effect of the top-level attribute 'allowedReferences'; use 'outputChecks' instead"); "'structuredAttrs' disables the effect of the top-level attribute 'allowedReferences'; use 'outputChecks' instead");
} }
if (get(*structuredAttrs, "allowedRequisites")) { if (get(parsed->structuredAttrs, "allowedRequisites")) {
warn( warn(
"'structuredAttrs' disables the effect of the top-level attribute 'allowedRequisites'; use 'outputChecks' instead"); "'structuredAttrs' disables the effect of the top-level attribute 'allowedRequisites'; use 'outputChecks' instead");
} }
if (get(*structuredAttrs, "disallowedRequisites")) { if (get(parsed->structuredAttrs, "disallowedRequisites")) {
warn( warn(
"'structuredAttrs' disables the effect of the top-level attribute 'disallowedRequisites'; use 'outputChecks' instead"); "'structuredAttrs' disables the effect of the top-level attribute 'disallowedRequisites'; use 'outputChecks' instead");
} }
if (get(*structuredAttrs, "disallowedReferences")) { if (get(parsed->structuredAttrs, "disallowedReferences")) {
warn( warn(
"'structuredAttrs' disables the effect of the top-level attribute 'disallowedReferences'; use 'outputChecks' instead"); "'structuredAttrs' disables the effect of the top-level attribute 'disallowedReferences'; use 'outputChecks' instead");
} }
if (get(*structuredAttrs, "maxSize")) { if (get(parsed->structuredAttrs, "maxSize")) {
warn( warn(
"'structuredAttrs' disables the effect of the top-level attribute 'maxSize'; use 'outputChecks' instead"); "'structuredAttrs' disables the effect of the top-level attribute 'maxSize'; use 'outputChecks' instead");
} }
if (get(*structuredAttrs, "maxClosureSize")) { if (get(parsed->structuredAttrs, "maxClosureSize")) {
warn( warn(
"'structuredAttrs' disables the effect of the top-level attribute 'maxClosureSize'; use 'outputChecks' instead"); "'structuredAttrs' disables the effect of the top-level attribute 'maxClosureSize'; use 'outputChecks' instead");
} }
@ -50,9 +126,9 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation
return { return {
.outputChecks = [&]() -> OutputChecksVariant { .outputChecks = [&]() -> OutputChecksVariant {
if (auto structuredAttrs = parsed.structuredAttrs.get()) { if (parsed) {
std::map<std::string, OutputChecks> res; std::map<std::string, OutputChecks> res;
if (auto outputChecks = get(*structuredAttrs, "outputChecks")) { if (auto outputChecks = get(parsed->structuredAttrs, "outputChecks")) {
for (auto & [outputName, output] : getObject(*outputChecks)) { for (auto & [outputName, output] : getObject(*outputChecks)) {
OutputChecks checks; OutputChecks checks;
@ -90,10 +166,10 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation
return OutputChecks{ return OutputChecks{
// legacy non-structured-attributes case // legacy non-structured-attributes case
.ignoreSelfRefs = true, .ignoreSelfRefs = true,
.allowedReferences = parsed.getStringSetAttr("allowedReferences"), .allowedReferences = getStringSetAttr(env, parsed, "allowedReferences"),
.disallowedReferences = parsed.getStringSetAttr("disallowedReferences").value_or(StringSet{}), .disallowedReferences = getStringSetAttr(env, parsed, "disallowedReferences").value_or(StringSet{}),
.allowedRequisites = parsed.getStringSetAttr("allowedRequisites"), .allowedRequisites = getStringSetAttr(env, parsed, "allowedRequisites"),
.disallowedRequisites = parsed.getStringSetAttr("disallowedRequisites").value_or(StringSet{}), .disallowedRequisites = getStringSetAttr(env, parsed, "disallowedRequisites").value_or(StringSet{}),
}; };
} }
}(), }(),
@ -101,8 +177,8 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation
[&] { [&] {
std::map<std::string, bool> res; std::map<std::string, bool> res;
if (auto structuredAttrs = parsed.structuredAttrs.get()) { if (parsed) {
if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) { if (auto udr = get(parsed->structuredAttrs, "unsafeDiscardReferences")) {
for (auto & [outputName, output] : getObject(*udr)) { for (auto & [outputName, output] : getObject(*udr)) {
if (!output.is_boolean()) if (!output.is_boolean())
throw Error("attribute 'unsafeDiscardReferences.\"%s\"' must be a Boolean", outputName); throw Error("attribute 'unsafeDiscardReferences.\"%s\"' must be a Boolean", outputName);
@ -116,8 +192,8 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation
.passAsFile = .passAsFile =
[&] { [&] {
StringSet res; StringSet res;
if (auto * passAsFileString = get(parsed.drv.env, "passAsFile")) { if (auto * passAsFileString = get(env, "passAsFile")) {
if (parsed.hasStructuredAttrs()) { if (parsed) {
if (shouldWarn) { if (shouldWarn) {
warn( warn(
"'structuredAttrs' disables the effect of the top-level attribute 'passAsFile'; because all JSON is always passed via file"); "'structuredAttrs' disables the effect of the top-level attribute 'passAsFile'; because all JSON is always passed via file");
@ -132,18 +208,18 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation
[&] { [&] {
std::map<std::string, StringSet> ret; std::map<std::string, StringSet> ret;
if (auto structuredAttrs = parsed.structuredAttrs.get()) { if (parsed) {
auto e = optionalValueAt(*structuredAttrs, "exportReferencesGraph"); auto e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph");
if (!e || !e->is_object()) if (!e || !e->is_object())
return ret; return ret;
for (auto & [key, storePathsJson] : getObject(*e)) { for (auto & [key, storePathsJson] : getObject(*e)) {
ret.insert_or_assign(key, storePathsJson); ret.insert_or_assign(key, storePathsJson);
} }
} else { } else {
auto s = getOr(parsed.drv.env, "exportReferencesGraph", ""); auto s = getOr(env, "exportReferencesGraph", "");
Strings ss = tokenizeString<Strings>(s); Strings ss = tokenizeString<Strings>(s);
if (ss.size() % 2 != 0) if (ss.size() % 2 != 0)
throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); throw Error("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
for (Strings::iterator i = ss.begin(); i != ss.end();) { for (Strings::iterator i = ss.begin(); i != ss.end();) {
auto fileName = std::move(*i++); auto fileName = std::move(*i++);
static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*");
@ -157,15 +233,15 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation
return ret; return ret;
}(), }(),
.additionalSandboxProfile = .additionalSandboxProfile =
parsed.getStringAttr("__sandboxProfile").value_or(defaults.additionalSandboxProfile), getStringAttr(env, parsed, "__sandboxProfile").value_or(defaults.additionalSandboxProfile),
.noChroot = parsed.getBoolAttr("__noChroot", defaults.noChroot), .noChroot = getBoolAttr(env, parsed, "__noChroot", defaults.noChroot),
.impureHostDeps = parsed.getStringSetAttr("__impureHostDeps").value_or(defaults.impureHostDeps), .impureHostDeps = getStringSetAttr(env, parsed, "__impureHostDeps").value_or(defaults.impureHostDeps),
.impureEnvVars = parsed.getStringSetAttr("impureEnvVars").value_or(defaults.impureEnvVars), .impureEnvVars = getStringSetAttr(env, parsed, "impureEnvVars").value_or(defaults.impureEnvVars),
.allowLocalNetworking = parsed.getBoolAttr("__darwinAllowLocalNetworking", defaults.allowLocalNetworking), .allowLocalNetworking = getBoolAttr(env, parsed, "__darwinAllowLocalNetworking", defaults.allowLocalNetworking),
.requiredSystemFeatures = .requiredSystemFeatures =
parsed.getStringSetAttr("requiredSystemFeatures").value_or(defaults.requiredSystemFeatures), getStringSetAttr(env, parsed, "requiredSystemFeatures").value_or(defaults.requiredSystemFeatures),
.preferLocalBuild = parsed.getBoolAttr("preferLocalBuild", defaults.preferLocalBuild), .preferLocalBuild = getBoolAttr(env, parsed, "preferLocalBuild", defaults.preferLocalBuild),
.allowSubstitutes = parsed.getBoolAttr("allowSubstitutes", defaults.allowSubstitutes), .allowSubstitutes = getBoolAttr(env, parsed, "allowSubstitutes", defaults.allowSubstitutes),
}; };
} }

3
src/libstore/include/nix/store/build/derivation-goal.hh
View file

@ -2,6 +2,7 @@
///@file ///@file
#include "nix/store/parsed-derivations.hh" #include "nix/store/parsed-derivations.hh"
#include "nix/store/derivations.hh"
#include "nix/store/derivation-options.hh" #include "nix/store/derivation-options.hh"
#ifndef _WIN32 #ifndef _WIN32
# include "nix/store/user-lock.hh" # include "nix/store/user-lock.hh"
@ -150,7 +151,7 @@ struct DerivationGoal : public Goal
*/ */
std::unique_ptr<Derivation> drv; std::unique_ptr<Derivation> drv;
std::unique_ptr<ParsedDerivation> parsedDrv; std::unique_ptr<StructuredAttrs> parsedDrv;
std::unique_ptr<DerivationOptions> drvOptions; std::unique_ptr<DerivationOptions> drvOptions;
/** /**

9
src/libstore/include/nix/store/derivation-options.hh
View file

@ -13,16 +13,16 @@ namespace nix {
class Store; class Store;
struct BasicDerivation; struct BasicDerivation;
class ParsedDerivation; struct StructuredAttrs;
/** /**
* This represents all the special options on a `Derivation`. * This represents all the special options on a `Derivation`.
* *
* Currently, these options are parsed from the environment variables * Currently, these options are parsed from the environment variables
* with the aid of `ParsedDerivation`. * with the aid of `StructuredAttrs`.
* *
* The first goal of this data type is to make sure that no other code * The first goal of this data type is to make sure that no other code
* uses `ParsedDerivation` to ad-hoc parse some additional options. That * uses `StructuredAttrs` to ad-hoc parse some additional options. That
* ensures this data type is up to date and fully correct. * ensures this data type is up to date and fully correct.
* *
* The second goal of this data type is to allow an alternative to * The second goal of this data type is to allow an alternative to
@ -173,7 +173,8 @@ struct DerivationOptions
* (e.g. JSON) but is necessary for supporing old formats (e.g. * (e.g. JSON) but is necessary for supporing old formats (e.g.
* ATerm). * ATerm).
*/ */
static DerivationOptions fromParsedDerivation(const ParsedDerivation & parsed, bool shouldWarn = true); static DerivationOptions
fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn = true);
/** /**
* @param drv Must be the same derivation we parsed this from. In * @param drv Must be the same derivation we parsed this from. In

58
src/libstore/include/nix/store/parsed-derivations.hh
View file

@ -1,51 +1,43 @@
#pragma once #pragma once
///@file ///@file
#include "nix/store/derivations.hh" #include <nlohmann/json.hpp>
#include "nix/store/store-api.hh"
#include <nlohmann/json_fwd.hpp> #include "nix/util/types.hh"
#include "nix/store/path.hh"
namespace nix { namespace nix {
class Store;
struct DerivationOptions; struct DerivationOptions;
struct DerivationOutput;
class ParsedDerivation typedef std::map<std::string, DerivationOutput> DerivationOutputs;
struct StructuredAttrs
{ {
BasicDerivation & drv; nlohmann::json structuredAttrs;
std::unique_ptr<nlohmann::json> structuredAttrs;
std::optional<std::string> getStringAttr(const std::string & name) const; static std::optional<StructuredAttrs> tryParse(const StringPairs & env);
bool getBoolAttr(const std::string & name, bool def = false) const; nlohmann::json prepareStructuredAttrs(
Store & store,
std::optional<Strings> getStringsAttr(const std::string & name) const; const DerivationOptions & drvOptions,
const StorePathSet & inputPaths,
std::optional<StringSet> getStringSetAttr(const std::string & name) const; const DerivationOutputs & outputs) const;
/** /**
* Only `DerivationOptions` is allowed to parse individual fields * As a convenience to bash scripts, write a shell file that
* from `ParsedDerivation`. This ensure that it includes all * maps all attributes that are representable in bash -
* derivation options, and, the likes of `LocalDerivationGoal` are * namely, strings, integers, nulls, Booleans, and arrays and
* incapable of more ad-hoc options. * objects consisting entirely of those values. (So nested
* arrays or objects are not supported.)
*
* @param prepared This should be the result of
* `prepareStructuredAttrs`, *not* the original `structuredAttrs`
* field.
*/ */
friend struct DerivationOptions; static std::string writeShell(const nlohmann::json & prepared);
public:
ParsedDerivation(BasicDerivation & drv);
~ParsedDerivation();
bool hasStructuredAttrs() const
{
return static_cast<bool>(structuredAttrs);
}
std::optional<nlohmann::json>
prepareStructuredAttrs(Store & store, const DerivationOptions & drvOptions, const StorePathSet & inputPaths);
}; };
std::string writeStructuredAttrsShell(const nlohmann::json & json);
} }

6
src/libstore/misc.cc
View file

@ -222,10 +222,12 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
if (knownOutputPaths && invalid.empty()) return; if (knownOutputPaths && invalid.empty()) return;
auto drv = make_ref<Derivation>(derivationFromPath(drvPath)); auto drv = make_ref<Derivation>(derivationFromPath(drvPath));
ParsedDerivation parsedDrv(*drv); auto parsedDrv = StructuredAttrs::tryParse(drv->env);
DerivationOptions drvOptions; DerivationOptions drvOptions;
try { try {
drvOptions = DerivationOptions::fromParsedDerivation(parsedDrv); drvOptions = DerivationOptions::fromStructuredAttrs(
drv->env,
parsedDrv ? &*parsedDrv : nullptr);
} catch (Error & e) { } catch (Error & e) {
e.addTrace({}, "while parsing derivation '%s'", printStorePath(drvPath)); e.addTrace({}, "while parsing derivation '%s'", printStorePath(drvPath));
throw; throw;

115
src/libstore/parsed-derivations.cc
View file

@ -1,4 +1,6 @@
#include "nix/store/parsed-derivations.hh" #include "nix/store/parsed-derivations.hh"
#include "nix/store/store-api.hh"
#include "nix/store/derivations.hh"
#include "nix/store/derivation-options.hh" #include "nix/store/derivation-options.hh"
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
@ -6,94 +8,20 @@
namespace nix { namespace nix {
ParsedDerivation::ParsedDerivation(BasicDerivation & drv) std::optional<StructuredAttrs> StructuredAttrs::tryParse(const StringPairs & env)
: drv(drv)
{ {
/* Parse the __json attribute, if any. */ /* Parse the __json attribute, if any. */
auto jsonAttr = drv.env.find("__json"); auto jsonAttr = env.find("__json");
if (jsonAttr != drv.env.end()) { if (jsonAttr != env.end()) {
try { try {
structuredAttrs = std::make_unique<nlohmann::json>(nlohmann::json::parse(jsonAttr->second)); return StructuredAttrs {
.structuredAttrs = nlohmann::json::parse(jsonAttr->second),
};
} catch (std::exception & e) { } catch (std::exception & e) {
throw Error("cannot process __json attribute: %s", e.what()); throw Error("cannot process __json attribute: %s", e.what());
} }
} }
} return {};
ParsedDerivation::~ParsedDerivation() { }
std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const
{
if (structuredAttrs) {
auto i = structuredAttrs->find(name);
if (i == structuredAttrs->end())
return {};
else {
if (!i->is_string())
throw Error("attribute '%s' of must be a string", name);
return i->get<std::string>();
}
} else {
auto i = drv.env.find(name);
if (i == drv.env.end())
return {};
else
return i->second;
}
}
bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const
{
if (structuredAttrs) {
auto i = structuredAttrs->find(name);
if (i == structuredAttrs->end())
return def;
else {
if (!i->is_boolean())
throw Error("attribute '%s' must be a Boolean", name);
return i->get<bool>();
}
} else {
auto i = drv.env.find(name);
if (i == drv.env.end())
return def;
else
return i->second == "1";
}
}
std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name) const
{
if (structuredAttrs) {
auto i = structuredAttrs->find(name);
if (i == structuredAttrs->end())
return {};
else {
if (!i->is_array())
throw Error("attribute '%s' must be a list of strings", name);
Strings res;
for (auto j = i->begin(); j != i->end(); ++j) {
if (!j->is_string())
throw Error("attribute '%s' must be a list of strings", name);
res.push_back(j->get<std::string>());
}
return res;
}
} else {
auto i = drv.env.find(name);
if (i == drv.env.end())
return {};
else
return tokenizeString<Strings>(i->second);
}
}
std::optional<StringSet> ParsedDerivation::getStringSetAttr(const std::string & name) const
{
auto ss = getStringsAttr(name);
return ss
? (std::optional{StringSet{ss->begin(), ss->end()}})
: (std::optional<StringSet>{});
} }
static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
@ -152,20 +80,20 @@ static nlohmann::json pathInfoToJSON(
return jsonList; return jsonList;
} }
std::optional<nlohmann::json> ParsedDerivation::prepareStructuredAttrs( nlohmann::json StructuredAttrs::prepareStructuredAttrs(
Store & store, Store & store,
const DerivationOptions & drvOptions, const DerivationOptions & drvOptions,
const StorePathSet & inputPaths) const StorePathSet & inputPaths,
const DerivationOutputs & outputs) const
{ {
if (!structuredAttrs) return std::nullopt; /* Copy to then modify */
auto json = structuredAttrs;
auto json = *structuredAttrs;
/* Add an "outputs" object containing the output paths. */ /* Add an "outputs" object containing the output paths. */
nlohmann::json outputs; nlohmann::json outputsJson;
for (auto & i : drv.outputs) for (auto & i : outputs)
outputs[i.first] = hashPlaceholder(i.first); outputsJson[i.first] = hashPlaceholder(i.first);
json["outputs"] = outputs; json["outputs"] = std::move(outputsJson);
/* Handle exportReferencesGraph. */ /* Handle exportReferencesGraph. */
for (auto & [key, inputPaths] : drvOptions.exportReferencesGraph) { for (auto & [key, inputPaths] : drvOptions.exportReferencesGraph) {
@ -179,12 +107,7 @@ std::optional<nlohmann::json> ParsedDerivation::prepareStructuredAttrs(
return json; return json;
} }
/* As a convenience to bash scripts, write a shell file that std::string StructuredAttrs::writeShell(const nlohmann::json & json)
maps all attributes that are representable in bash -
namely, strings, integers, nulls, Booleans, and arrays and
objects consisting entirely of those values. (So nested
arrays or objects are not supported.) */
std::string writeStructuredAttrsShell(const nlohmann::json & json)
{ {
auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> { auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {

14
src/libstore/unix/build/local-derivation-goal.cc
View file

@ -971,7 +971,7 @@ void LocalDerivationGoal::startBuilder()
writeStructuredAttrs(); writeStructuredAttrs();
/* Handle exportReferencesGraph(), if set. */ /* Handle exportReferencesGraph(), if set. */
if (!parsedDrv->hasStructuredAttrs()) { if (!parsedDrv) {
for (auto & [fileName, ss] : drvOptions->exportReferencesGraph) { for (auto & [fileName, ss] : drvOptions->exportReferencesGraph) {
StorePathSet storePathSet; StorePathSet storePathSet;
for (auto & storePathS : ss) { for (auto & storePathS : ss) {
@ -1475,7 +1475,7 @@ void LocalDerivationGoal::initTmpDir()
/* In non-structured mode, set all bindings either directory in the /* In non-structured mode, set all bindings either directory in the
environment or via a file, as specified by environment or via a file, as specified by
`DerivationOptions::passAsFile`. */ `DerivationOptions::passAsFile`. */
if (!parsedDrv->hasStructuredAttrs()) { if (!parsedDrv) {
for (auto & i : drv->env) { for (auto & i : drv->env) {
if (drvOptions->passAsFile.find(i.first) == drvOptions->passAsFile.end()) { if (drvOptions->passAsFile.find(i.first) == drvOptions->passAsFile.end()) {
env[i.first] = i.second; env[i.first] = i.second;
@ -1576,8 +1576,12 @@ void LocalDerivationGoal::initEnv()
void LocalDerivationGoal::writeStructuredAttrs() void LocalDerivationGoal::writeStructuredAttrs()
{ {
if (auto structAttrsJson = parsedDrv->prepareStructuredAttrs(worker.store, *drvOptions, inputPaths)) { if (parsedDrv) {
auto json = structAttrsJson.value(); auto json = parsedDrv->prepareStructuredAttrs(
worker.store,
*drvOptions,
inputPaths,
drv->outputs);
nlohmann::json rewritten; nlohmann::json rewritten;
for (auto & [i, v] : json["outputs"].get<nlohmann::json::object_t>()) { for (auto & [i, v] : json["outputs"].get<nlohmann::json::object_t>()) {
/* The placeholder must have a rewrite, so we use it to cover both the /* The placeholder must have a rewrite, so we use it to cover both the
@ -1587,7 +1591,7 @@ void LocalDerivationGoal::writeStructuredAttrs()
json["outputs"] = rewritten; json["outputs"] = rewritten;
auto jsonSh = writeStructuredAttrsShell(json); auto jsonSh = StructuredAttrs::writeShell(json);
writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
chownToBuilder(tmpDir + "/.attrs.sh"); chownToBuilder(tmpDir + "/.attrs.sh");

31
src/nix-build/nix-build.cc
View file

@ -544,10 +544,12 @@ static void main_nix_build(int argc, char * * argv)
env["NIX_STORE"] = store->storeDir; env["NIX_STORE"] = store->storeDir;
env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores); env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores);
ParsedDerivation parsedDrv(drv); auto parsedDrv = StructuredAttrs::tryParse(drv.env);
DerivationOptions drvOptions; DerivationOptions drvOptions;
try { try {
drvOptions = DerivationOptions::fromParsedDerivation(parsedDrv); drvOptions = DerivationOptions::fromStructuredAttrs(
drv.env,
parsedDrv ? &*parsedDrv : nullptr);
} catch (Error & e) { } catch (Error & e) {
e.addTrace({}, "while parsing derivation '%s'", store->printStorePath(packageInfo.requireDrvPath())); e.addTrace({}, "while parsing derivation '%s'", store->printStorePath(packageInfo.requireDrvPath()));
throw; throw;
@ -566,7 +568,7 @@ static void main_nix_build(int argc, char * * argv)
std::string structuredAttrsRC; std::string structuredAttrsRC;
if (parsedDrv.hasStructuredAttrs()) { if (parsedDrv) {
StorePathSet inputs; StorePathSet inputs;
std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputClosure; std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputClosure;
@ -584,19 +586,22 @@ static void main_nix_build(int argc, char * * argv)
for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
accumInputClosure(inputDrv, inputNode); accumInputClosure(inputDrv, inputNode);
if (auto structAttrs = parsedDrv.prepareStructuredAttrs(*store, drvOptions, inputs)) { auto json = parsedDrv->prepareStructuredAttrs(
auto json = structAttrs.value(); *store,
structuredAttrsRC = writeStructuredAttrsShell(json); drvOptions,
inputs,
drv.outputs);
auto attrsJSON = (tmpDir.path() / ".attrs.json").string(); structuredAttrsRC = StructuredAttrs::writeShell(json);
writeFile(attrsJSON, json.dump());
auto attrsSH = (tmpDir.path() / ".attrs.sh").string(); auto attrsJSON = (tmpDir.path() / ".attrs.json").string();
writeFile(attrsSH, structuredAttrsRC); writeFile(attrsJSON, json.dump());
env["NIX_ATTRS_SH_FILE"] = attrsSH; auto attrsSH = (tmpDir.path() / ".attrs.sh").string();
env["NIX_ATTRS_JSON_FILE"] = attrsJSON; writeFile(attrsSH, structuredAttrsRC);
}
env["NIX_ATTRS_SH_FILE"] = attrsSH;
env["NIX_ATTRS_JSON_FILE"] = attrsJSON;
} }
/* Run a shell using the derivation's environment. For /* Run a shell using the derivation's environment. For