diff --git a/.version b/.version index 9738a24f6..0bd6cbc1e 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.28.1 +2.28.2 diff --git a/doc/manual/source/language/string-context.md b/doc/manual/source/language/string-context.md index 6a3482cfd..979bbf371 100644 --- a/doc/manual/source/language/string-context.md +++ b/doc/manual/source/language/string-context.md @@ -115,7 +115,7 @@ It creates an [attribute set] representing the string context, which can be insp ## Clearing string contexts -[`buitins.unsafeDiscardStringContext`](./builtins.md#builtins-unsafeDiscardStringContext) will make a copy of a string, but with an empty string context. +[`builtins.unsafeDiscardStringContext`](./builtins.md#builtins-unsafeDiscardStringContext) will make a copy of a string, but with an empty string context. The returned string can be used in more ways, e.g. by operators that require the string context to be empty. The requirement to explicitly discard the string context in such use cases helps ensure that string context elements are not lost by mistake. The "unsafe" marker is only there to remind that Nix normally guarantees that dependencies are tracked, whereas the returned string has lost them. diff --git a/doc/manual/source/release-notes/rl-2.27.md b/doc/manual/source/release-notes/rl-2.27.md index b4918029a..3643f7476 100644 --- a/doc/manual/source/release-notes/rl-2.27.md +++ b/doc/manual/source/release-notes/rl-2.27.md @@ -38,6 +38,15 @@ Curl created sockets without setting `FD_CLOEXEC`/`SOCK_CLOEXEC`. This could previously cause connections to remain open forever when using commands like `nix shell`. This change sets the `FD_CLOEXEC` flag using a `CURLOPT_SOCKOPTFUNCTION` callback. +- Add BLAKE3 hash algorithm [#12379](https://github.com/NixOS/nix/pull/12379) + + Nix now supports the BLAKE3 hash algorithm as an experimental feature (`blake3-hashes`): + + ```console + # nix hash file ./file --type blake3 --extra-experimental-features blake3-hashes + blake3-34P4p+iZXcbbyB1i4uoF7eWCGcZHjmaRn6Y7QdynLwU= + ``` + # Contributors This release was made possible by the following 21 contributors: diff --git a/maintainers/release-notes b/maintainers/release-notes index 0cdcd517b..6586b22dc 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -2,6 +2,8 @@ # vim: set filetype=bash: #!nix shell .#changelog-d --command bash +set -euo pipefail + # --- CONFIGURATION --- # This does double duty for diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 565f424dd..56541fa57 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -237,12 +237,13 @@ void StorePathCommand::run(ref store, StorePaths && storePaths) MixProfile::MixProfile() { - addFlag( - {.longName = "profile", - .description = "The profile to operate on.", - .labels = {"path"}, - .handler = {&profile}, - .completer = completePath}); + addFlag({ + .longName = "profile", + .description = "The profile to operate on.", + .labels = {"path"}, + .handler = {&profile}, + .completer = completePath, + }); } void MixProfile::updateProfile(const StorePath & storePath) diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index bd8e3945b..21584b74a 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -62,7 +62,7 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the value *expr* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "expr"}, - .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr{expr}}); }} + .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr{expr}}); }}, }); addFlag({ @@ -79,7 +79,7 @@ MixEvalArgs::MixEvalArgs() .category = category, .labels = {"name", "path"}, .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile{path}}); }}, - .completer = completePath + .completer = completePath, }); addFlag({ @@ -104,7 +104,7 @@ MixEvalArgs::MixEvalArgs() .labels = {"path"}, .handler = {[&](std::string s) { lookupPath.elements.emplace_back(LookupPath::Elem::parse(s)); - }} + }}, }); addFlag({ @@ -130,7 +130,7 @@ MixEvalArgs::MixEvalArgs() }}, .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { completeFlakeRef(completions, openStore(), prefix); - }} + }}, }); addFlag({ diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index e4a1d0a42..edfe8c15a 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -64,21 +64,21 @@ MixFlakeOptions::MixFlakeOptions() .handler = {[&]() { lockFlags.recreateLockFile = true; warn("'--recreate-lock-file' is deprecated and will be removed in a future version; use 'nix flake update' instead."); - }} + }}, }); addFlag({ .longName = "no-update-lock-file", .description = "Do not allow any updates to the flake's lock file.", .category = category, - .handler = {&lockFlags.updateLockFile, false} + .handler = {&lockFlags.updateLockFile, false}, }); addFlag({ .longName = "no-write-lock-file", .description = "Do not write the flake's newly generated lock file.", .category = category, - .handler = {&lockFlags.writeLockFile, false} + .handler = {&lockFlags.writeLockFile, false}, }); addFlag({ @@ -94,14 +94,14 @@ MixFlakeOptions::MixFlakeOptions() .handler = {[&]() { lockFlags.useRegistries = false; warn("'--no-registries' is deprecated; use '--no-use-registries'"); - }} + }}, }); addFlag({ .longName = "commit-lock-file", .description = "Commit changes to the flake's lock file.", .category = category, - .handler = {&lockFlags.commitLockFile, true} + .handler = {&lockFlags.commitLockFile, true}, }); addFlag({ @@ -121,7 +121,7 @@ MixFlakeOptions::MixFlakeOptions() }}, .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { completeFlakeInputAttrPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); - }} + }}, }); addFlag({ @@ -141,7 +141,7 @@ MixFlakeOptions::MixFlakeOptions() } else if (n == 1) { completeFlakeRef(completions, getEvalState()->store, prefix); } - }} + }}, }); addFlag({ @@ -152,7 +152,7 @@ MixFlakeOptions::MixFlakeOptions() .handler = {[&](std::string lockFilePath) { lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath))}; }}, - .completer = completePath + .completer = completePath, }); addFlag({ @@ -163,7 +163,7 @@ MixFlakeOptions::MixFlakeOptions() .handler = {[&](std::string lockFilePath) { lockFlags.outputLockFilePath = lockFilePath; }}, - .completer = completePath + .completer = completePath, }); addFlag({ @@ -190,7 +190,7 @@ MixFlakeOptions::MixFlakeOptions() }}, .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { completeFlakeRef(completions, getEvalState()->store, prefix); - }} + }}, }); } @@ -206,7 +206,7 @@ SourceExprCommand::SourceExprCommand() .category = installablesCategory, .labels = {"file"}, .handler = {&file}, - .completer = completePath + .completer = completePath, }); addFlag({ @@ -214,7 +214,7 @@ SourceExprCommand::SourceExprCommand() .description = "Interpret [*installables*](@docroot@/command-ref/new-cli/nix.md#installables) as attribute paths relative to the Nix expression *expr*.", .category = installablesCategory, .labels = {"expr"}, - .handler = {&expr} + .handler = {&expr}, }); } @@ -831,7 +831,7 @@ RawInstallablesCommand::RawInstallablesCommand() addFlag({ .longName = "stdin", .description = "Read installables from the standard input. No default installable applied.", - .handler = {&readFromStdIn, true} + .handler = {&readFromStdIn, true}, }); expectArgs({ diff --git a/src/libexpr/include/nix/expr/meson.build b/src/libexpr/include/nix/expr/meson.build index 01275e52e..50ea8f3c2 100644 --- a/src/libexpr/include/nix/expr/meson.build +++ b/src/libexpr/include/nix/expr/meson.build @@ -20,7 +20,6 @@ headers = [config_pub_h] + files( 'gc-small-vector.hh', 'get-drvs.hh', 'json-to-value.hh', - # internal: 'lexer-helpers.hh', 'nixexpr.hh', 'parser-state.hh', 'primops.hh', diff --git a/src/libexpr/include/nix/expr/nixexpr.hh b/src/libexpr/include/nix/expr/nixexpr.hh index 9409bdca8..a5ce0fd89 100644 --- a/src/libexpr/include/nix/expr/nixexpr.hh +++ b/src/libexpr/include/nix/expr/nixexpr.hh @@ -65,7 +65,7 @@ struct DocComment { struct AttrName { Symbol symbol; - Expr * expr; + Expr * expr = nullptr; AttrName(Symbol s) : symbol(s) {}; AttrName(Expr * e) : expr(e) {}; }; @@ -159,7 +159,7 @@ struct ExprVar : Expr `nullptr`: Not from a `with`. Valid pointer: the nearest, innermost `with` expression to query first. */ - ExprWith * fromWith; + ExprWith * fromWith = nullptr; /* In the former case, the value is obtained by going `level` levels up from the current environment and getting the @@ -167,7 +167,7 @@ struct ExprVar : Expr value is obtained by getting the attribute named `name` from the set stored in the environment that is `level` levels up from the current one.*/ - Level level; + Level level = 0; Displacement displ = 0; ExprVar(Symbol name) : name(name) { }; diff --git a/src/libexpr/lexer-helpers.cc b/src/libexpr/lexer-helpers.cc index 4b27393bb..927e3cc73 100644 --- a/src/libexpr/lexer-helpers.cc +++ b/src/libexpr/lexer-helpers.cc @@ -1,7 +1,4 @@ -#include "lexer-tab.hh" -#include "parser-tab.hh" - -#include "nix/expr/lexer-helpers.hh" +#include "lexer-helpers.hh" void nix::lexer::internal::initLoc(YYLTYPE * loc) { diff --git a/src/libexpr/include/nix/expr/lexer-helpers.hh b/src/libexpr/lexer-helpers.hh similarity index 100% rename from src/libexpr/include/nix/expr/lexer-helpers.hh rename to src/libexpr/lexer-helpers.hh diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 511c8e47b..1e196741d 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -25,8 +25,7 @@ #endif #include "nix/expr/nixexpr.hh" -#include "parser-tab.hh" -#include "nix/expr/lexer-helpers.hh" +#include "lexer-helpers.hh" namespace nix { struct LexerState; diff --git a/src/libflake/call-flake.nix b/src/libflake/call-flake.nix index 1e9e21048..fe326291f 100644 --- a/src/libflake/call-flake.nix +++ b/src/libflake/call-flake.nix @@ -14,6 +14,7 @@ overrides: fetchTreeFinal: let + inherit (builtins) mapAttrs; lockFile = builtins.fromJSON lockFileStr; @@ -35,19 +36,26 @@ let (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path}) (builtins.tail path); - allNodes = builtins.mapAttrs ( + allNodes = mapAttrs ( key: node: let parentNode = allNodes.${getInputByPath lockFile.root node.parent}; + flakeDir = + let + dir = overrides.${key}.dir or node.locked.path or ""; + parentDir = parentNode.flakeDir; + in + if node ? parent then parentDir + ("/" + dir) else dir; + sourceInfo = if overrides ? ${key} then overrides.${key}.sourceInfo else if node.locked.type == "path" && builtins.substring 0 1 node.locked.path != "/" then parentNode.sourceInfo // { - outPath = parentNode.outPath + ("/" + node.locked.path); + outPath = parentNode.sourceInfo.outPath + ("/" + flakeDir); } else # FIXME: remove obsolete node.info. @@ -60,7 +68,7 @@ let flake = import (outPath + "/flake.nix"); - inputs = builtins.mapAttrs (inputName: inputSpec: allNodes.${resolveInput inputSpec}) ( + inputs = mapAttrs (inputName: inputSpec: allNodes.${resolveInput inputSpec}.result) ( node.inputs or { } ); @@ -85,12 +93,17 @@ let }; in - if node.flake or true then - assert builtins.isFunction flake.outputs; - result - else - sourceInfo + { + result = + if node.flake or true then + assert builtins.isFunction flake.outputs; + result + else + sourceInfo; + + inherit flakeDir sourceInfo; + } ) lockFile.nodes; in -allNodes.${lockFile.root} +allNodes.${lockFile.root}.result diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 6e95eb767..1580c2846 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -39,7 +39,7 @@ FlakeRef FlakeRef::resolve( ref store, const fetchers::RegistryFilter & filter) const { - auto [input2, extraAttrs] = lookupInRegistries(store, input); + auto [input2, extraAttrs] = lookupInRegistries(store, input, filter); return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); } diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index c3338996c..13b85e544 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -57,7 +57,7 @@ MixCommonArgs::MixCommonArgs(const std::string & programName) if (hasPrefix(s.first, prefix)) completions.add(s.first, fmt("Set the `%s` setting.", s.first)); } - } + }, }); addFlag({ @@ -75,7 +75,7 @@ MixCommonArgs::MixCommonArgs(const std::string & programName) .labels = Strings{"jobs"}, .handler = {[=](std::string s) { settings.set("max-jobs", s); - }} + }}, }); std::string cat = "Options to override configuration settings"; diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 62bd3b1d3..d9e8059f7 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -231,7 +231,7 @@ LegacyArgs::LegacyArgs(const std::string & programName, .handler = {[=](std::string s) { auto n = string2IntWithUnitPrefix(s); settings.set(dest, std::to_string(n)); - }} + }}, }); }; diff --git a/src/libstore-tests/data/derivation/advanced-attributes-defaults.drv b/src/libstore-tests/data/derivation/advanced-attributes-defaults.drv deleted file mode 120000 index f8f30ac32..000000000 --- a/src/libstore-tests/data/derivation/advanced-attributes-defaults.drv +++ /dev/null @@ -1 +0,0 @@ -../../../../tests/functional/derivation/advanced-attributes-defaults.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs-defaults.drv b/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs-defaults.drv deleted file mode 120000 index 837e9a0e4..000000000 --- a/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs-defaults.drv +++ /dev/null @@ -1 +0,0 @@ -../../../../tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs.drv b/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs.drv deleted file mode 120000 index e08bb5737..000000000 --- a/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs.drv +++ /dev/null @@ -1 +0,0 @@ -../../../../tests/functional/derivation/advanced-attributes-structured-attrs.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs.json deleted file mode 100644 index 324428124..000000000 --- a/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "args": [ - "-c", - "echo hello > $out" - ], - "builder": "/bin/bash", - "env": { - "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"],\"disallowedRequisites\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"allowedRequisites\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", - "bin": "/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin", - "dev": "/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev", - "out": "/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs" - }, - "inputDrvs": { - "/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv": { - "dynamicOutputs": {}, - "outputs": [ - "out" - ] - }, - "/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv": { - "dynamicOutputs": {}, - "outputs": [ - "out" - ] - } - }, - "inputSrcs": [], - "name": "advanced-attributes-structured-attrs", - "outputs": { - "bin": { - "path": "/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin" - }, - "dev": { - "path": "/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev" - }, - "out": { - "path": "/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs" - } - }, - "system": "my-system" -} diff --git a/src/libstore-tests/data/derivation/advanced-attributes.drv b/src/libstore-tests/data/derivation/advanced-attributes.drv deleted file mode 120000 index 1dc394a0a..000000000 --- a/src/libstore-tests/data/derivation/advanced-attributes.drv +++ /dev/null @@ -1 +0,0 @@ -../../../../tests/functional/derivation/advanced-attributes.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.drv b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.drv new file mode 120000 index 000000000..a9b4f7fa7 --- /dev/null +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.drv @@ -0,0 +1 @@ +../../../../../tests/functional/derivation/ca/advanced-attributes-defaults.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json new file mode 100644 index 000000000..bc67236b5 --- /dev/null +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json @@ -0,0 +1,25 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "builder": "/bin/bash", + "name": "advanced-attributes-defaults", + "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9", + "outputHashAlgo": "sha256", + "outputHashMode": "recursive", + "system": "my-system" + }, + "inputDrvs": {}, + "inputSrcs": [], + "name": "advanced-attributes-defaults", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "my-system" +} diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.drv b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.drv new file mode 120000 index 000000000..61da0470a --- /dev/null +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.drv @@ -0,0 +1 @@ +../../../../../tests/functional/derivation/ca/advanced-attributes-structured-attrs-defaults.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json new file mode 100644 index 000000000..7d3c932b2 --- /dev/null +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json @@ -0,0 +1,26 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "__json": "{\"builder\":\"/bin/bash\",\"name\":\"advanced-attributes-structured-attrs-defaults\",\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"dev\"],\"system\":\"my-system\"}", + "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz", + "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9" + }, + "inputDrvs": {}, + "inputSrcs": [], + "name": "advanced-attributes-structured-attrs-defaults", + "outputs": { + "dev": { + "hashAlgo": "sha256", + "method": "nar" + }, + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "my-system" +} diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.drv b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.drv new file mode 120000 index 000000000..c396ee853 --- /dev/null +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.drv @@ -0,0 +1 @@ +../../../../../tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json new file mode 100644 index 000000000..f6cdc1f16 --- /dev/null +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json @@ -0,0 +1,46 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g\"],\"disallowedRequisites\":[\"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"allowedRequisites\":[\"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", + "bin": "/04f3da1kmbr67m3gzxikmsl4vjz5zf777sv6m14ahv22r65aac9m", + "dev": "/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz", + "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9" + }, + "inputDrvs": { + "/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + }, + "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + } + }, + "inputSrcs": [], + "name": "advanced-attributes-structured-attrs", + "outputs": { + "bin": { + "hashAlgo": "sha256", + "method": "nar" + }, + "dev": { + "hashAlgo": "sha256", + "method": "nar" + }, + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "my-system" +} diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes.drv b/src/libstore-tests/data/derivation/ca/advanced-attributes.drv new file mode 120000 index 000000000..acba9064d --- /dev/null +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes.drv @@ -0,0 +1 @@ +../../../../../tests/functional/derivation/ca/advanced-attributes.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes.json b/src/libstore-tests/data/derivation/ca/advanced-attributes.json new file mode 100644 index 000000000..2105c6256 --- /dev/null +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes.json @@ -0,0 +1,52 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "__darwinAllowLocalNetworking": "1", + "__impureHostDeps": "/usr/bin/ditto", + "__noChroot": "1", + "__sandboxProfile": "sandcastle", + "allowSubstitutes": "", + "allowedReferences": "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9", + "allowedRequisites": "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z", + "builder": "/bin/bash", + "disallowedReferences": "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g", + "disallowedRequisites": "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8", + "impureEnvVars": "UNICORN", + "name": "advanced-attributes", + "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9", + "outputHashAlgo": "sha256", + "outputHashMode": "recursive", + "preferLocalBuild": "1", + "requiredSystemFeatures": "rainbow uid-range", + "system": "my-system" + }, + "inputDrvs": { + "/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + }, + "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + } + }, + "inputSrcs": [], + "name": "advanced-attributes", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "my-system" +} diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.drv b/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.drv new file mode 120000 index 000000000..7f1aa367e --- /dev/null +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.drv @@ -0,0 +1 @@ +../../../../../tests/functional/derivation/ia/advanced-attributes-defaults.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/advanced-attributes-defaults.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json similarity index 100% rename from src/libstore-tests/data/derivation/advanced-attributes-defaults.json rename to src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.drv b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.drv new file mode 120000 index 000000000..77aa67353 --- /dev/null +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.drv @@ -0,0 +1 @@ +../../../../../tests/functional/derivation/ia/advanced-attributes-structured-attrs-defaults.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs-defaults.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json similarity index 100% rename from src/libstore-tests/data/derivation/advanced-attributes-structured-attrs-defaults.json rename to src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.drv b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.drv new file mode 120000 index 000000000..a4e25feba --- /dev/null +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.drv @@ -0,0 +1 @@ +../../../../../tests/functional/derivation/ia/advanced-attributes-structured-attrs.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json new file mode 100644 index 000000000..b45a0d624 --- /dev/null +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json @@ -0,0 +1,43 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar\"],\"disallowedRequisites\":[\"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"allowedRequisites\":[\"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", + "bin": "/nix/store/qjjj3zrlimpjbkk686m052b3ks9iz2sl-advanced-attributes-structured-attrs-bin", + "dev": "/nix/store/lpz5grl48v93pdadavyg5is1rqvfdipf-advanced-attributes-structured-attrs-dev", + "out": "/nix/store/nzvz1bmh1g89a5dkpqcqan0av7q3hgv3-advanced-attributes-structured-attrs" + }, + "inputDrvs": { + "/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + }, + "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + } + }, + "inputSrcs": [], + "name": "advanced-attributes-structured-attrs", + "outputs": { + "bin": { + "path": "/nix/store/qjjj3zrlimpjbkk686m052b3ks9iz2sl-advanced-attributes-structured-attrs-bin" + }, + "dev": { + "path": "/nix/store/lpz5grl48v93pdadavyg5is1rqvfdipf-advanced-attributes-structured-attrs-dev" + }, + "out": { + "path": "/nix/store/nzvz1bmh1g89a5dkpqcqan0av7q3hgv3-advanced-attributes-structured-attrs" + } + }, + "system": "my-system" +} diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes.drv b/src/libstore-tests/data/derivation/ia/advanced-attributes.drv new file mode 120000 index 000000000..ecc2f5f38 --- /dev/null +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes.drv @@ -0,0 +1 @@ +../../../../../tests/functional/derivation/ia/advanced-attributes.drv \ No newline at end of file diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes.json b/src/libstore-tests/data/derivation/ia/advanced-attributes.json new file mode 100644 index 000000000..1eb8de86e --- /dev/null +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes.json @@ -0,0 +1,49 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "__darwinAllowLocalNetworking": "1", + "__impureHostDeps": "/usr/bin/ditto", + "__noChroot": "1", + "__sandboxProfile": "sandcastle", + "allowSubstitutes": "", + "allowedReferences": "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo", + "allowedRequisites": "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev", + "builder": "/bin/bash", + "disallowedReferences": "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar", + "disallowedRequisites": "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev", + "impureEnvVars": "UNICORN", + "name": "advanced-attributes", + "out": "/nix/store/swkj0mrq0cq3dfli95v4am0427mi2hxf-advanced-attributes", + "preferLocalBuild": "1", + "requiredSystemFeatures": "rainbow uid-range", + "system": "my-system" + }, + "inputDrvs": { + "/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + }, + "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "dev", + "out" + ] + } + }, + "inputSrcs": [], + "name": "advanced-attributes", + "outputs": { + "out": { + "path": "/nix/store/swkj0mrq0cq3dfli95v4am0427mi2hxf-advanced-attributes" + } + }, + "system": "my-system" +} diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index 57b226826..f82cea026 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -18,68 +18,93 @@ using nlohmann::json; class DerivationAdvancedAttrsTest : public CharacterizationTest, public LibStoreTest { - std::filesystem::path unitTestData = getUnitTestData() / "derivation"; +protected: + std::filesystem::path unitTestData = getUnitTestData() / "derivation" / "ia"; public: std::filesystem::path goldenMaster(std::string_view testStem) const override { return unitTestData / testStem; } + + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; }; -#define TEST_ATERM_JSON(STEM, NAME) \ - TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_from_json) \ - { \ - readTest(NAME ".json", [&](const auto & encoded_) { \ - auto encoded = json::parse(encoded_); \ - /* Use DRV file instead of C++ literal as source of truth. */ \ - auto aterm = readFile(goldenMaster(NAME ".drv")); \ - auto expected = parseDerivation(*store, std::move(aterm), NAME); \ - Derivation got = Derivation::fromJSON(*store, encoded); \ - EXPECT_EQ(got, expected); \ - }); \ - } \ - \ - TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_to_json) \ - { \ - writeTest( \ - NAME ".json", \ - [&]() -> json { \ - /* Use DRV file instead of C++ literal as source of truth. */ \ - auto aterm = readFile(goldenMaster(NAME ".drv")); \ - return parseDerivation(*store, std::move(aterm), NAME).toJSON(*store); \ - }, \ - [](const auto & file) { return json::parse(readFile(file)); }, \ - [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ - } \ - \ - TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_from_aterm) \ - { \ - readTest(NAME ".drv", [&](auto encoded) { \ - /* Use JSON file instead of C++ literal as source of truth. */ \ - auto json = json::parse(readFile(goldenMaster(NAME ".json"))); \ - auto expected = Derivation::fromJSON(*store, json); \ - auto got = parseDerivation(*store, std::move(encoded), NAME); \ - EXPECT_EQ(got.toJSON(*store), expected.toJSON(*store)); \ - EXPECT_EQ(got, expected); \ - }); \ - } \ - \ +class CaDerivationAdvancedAttrsTest : public DerivationAdvancedAttrsTest +{ + void SetUp() override + { + unitTestData = getUnitTestData() / "derivation" / "ca"; + mockXpSettings.set("experimental-features", "ca-derivations"); + } +}; + +template +class DerivationAdvancedAttrsBothTest : public Fixture +{}; + +using BothFixtures = ::testing::Types; + +TYPED_TEST_SUITE(DerivationAdvancedAttrsBothTest, BothFixtures); + +#define TEST_ATERM_JSON(STEM, NAME) \ + TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_json) \ + { \ + this->readTest(NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ + auto expected = parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings); \ + Derivation got = Derivation::fromJSON(*this->store, encoded, this->mockXpSettings); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ + TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_to_json) \ + { \ + this->writeTest( \ + NAME ".json", \ + [&]() -> json { \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ + return parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings) \ + .toJSON(*this->store); \ + }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } \ + \ + TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_aterm) \ + { \ + this->readTest(NAME ".drv", [&](auto encoded) { \ + /* Use JSON file instead of C++ literal as source of truth. */ \ + auto json = json::parse(readFile(this->goldenMaster(NAME ".json"))); \ + auto expected = Derivation::fromJSON(*this->store, json, this->mockXpSettings); \ + auto got = parseDerivation(*this->store, std::move(encoded), NAME, this->mockXpSettings); \ + EXPECT_EQ(got.toJSON(*this->store), expected.toJSON(*this->store)); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ /* No corresponding write test, because we need to read the drv to write the json file */ -TEST_ATERM_JSON(advancedAttributes_defaults, "advanced-attributes-defaults"); TEST_ATERM_JSON(advancedAttributes, "advanced-attributes-defaults"); -TEST_ATERM_JSON(advancedAttributes_structuredAttrs_defaults, "advanced-attributes-structured-attrs"); +TEST_ATERM_JSON(advancedAttributes_defaults, "advanced-attributes"); TEST_ATERM_JSON(advancedAttributes_structuredAttrs, "advanced-attributes-structured-attrs-defaults"); +TEST_ATERM_JSON(advancedAttributes_structuredAttrs_defaults, "advanced-attributes-structured-attrs"); #undef TEST_ATERM_JSON -TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_defaults) +TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_defaults) { - readTest("advanced-attributes-defaults.drv", [&](auto encoded) { - auto got = parseDerivation(*store, std::move(encoded), "foo"); + this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - auto drvPath = writeDerivation(*store, got, NoRepair, true); + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); ParsedDerivation parsedDrv(drvPath, got); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); @@ -101,25 +126,50 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_defaults) EXPECT_EQ(checksForAllOutputs.disallowedReferences, StringSet{}); EXPECT_EQ(checksForAllOutputs.disallowedRequisites, StringSet{}); } - EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet()); - EXPECT_EQ(options.canBuildLocally(*store, got), false); - EXPECT_EQ(options.willBuildLocally(*store, got), false); + EXPECT_EQ(options.canBuildLocally(*this->store, got), false); + EXPECT_EQ(options.willBuildLocally(*this->store, got), false); EXPECT_EQ(options.substitutesAllowed(), true); EXPECT_EQ(options.useUidRange(got), false); }); }; -TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes) +TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_defaults) { - readTest("advanced-attributes.drv", [&](auto encoded) { - auto got = parseDerivation(*store, std::move(encoded), "foo"); + this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - auto drvPath = writeDerivation(*store, got, NoRepair, true); + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); ParsedDerivation parsedDrv(drvPath, got); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); - StringSet systemFeatures{"rainbow", "uid-range"}; + EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{}); + }); +}; + +TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_defaults) +{ + this->readTest("advanced-attributes-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + + EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"}); + }); +}; + +TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes) +{ + this->readTest("advanced-attributes.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); EXPECT_TRUE(!parsedDrv.hasStructuredAttrs()); @@ -128,34 +178,88 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes) EXPECT_EQ(options.impureHostDeps, StringSet{"/usr/bin/ditto"}); EXPECT_EQ(options.impureEnvVars, StringSet{"UNICORN"}); EXPECT_EQ(options.allowLocalNetworking, true); + EXPECT_EQ(options.canBuildLocally(*this->store, got), false); + EXPECT_EQ(options.willBuildLocally(*this->store, got), false); + EXPECT_EQ(options.substitutesAllowed(), false); + EXPECT_EQ(options.useUidRange(got), true); + }); +}; + +TEST_F(DerivationAdvancedAttrsTest, advancedAttributes) +{ + this->readTest("advanced-attributes.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + { auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks); ASSERT_TRUE(checksForAllOutputs_ != nullptr); auto & checksForAllOutputs = *checksForAllOutputs_; EXPECT_EQ( - checksForAllOutputs.allowedReferences, StringSet{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + checksForAllOutputs.allowedReferences, StringSet{"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}); EXPECT_EQ( - checksForAllOutputs.allowedRequisites, StringSet{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + checksForAllOutputs.allowedRequisites, + StringSet{"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"}); EXPECT_EQ( - checksForAllOutputs.disallowedReferences, StringSet{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + checksForAllOutputs.disallowedReferences, StringSet{"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"}); EXPECT_EQ( - checksForAllOutputs.disallowedRequisites, StringSet{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + checksForAllOutputs.disallowedRequisites, + StringSet{"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"}); } + + StringSet systemFeatures{"rainbow", "uid-range"}; + EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures); - EXPECT_EQ(options.canBuildLocally(*store, got), false); - EXPECT_EQ(options.willBuildLocally(*store, got), false); - EXPECT_EQ(options.substitutesAllowed(), false); - EXPECT_EQ(options.useUidRange(got), true); }); }; -TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttrs_defaults) +TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes) { - readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) { - auto got = parseDerivation(*store, std::move(encoded), "foo"); + this->readTest("advanced-attributes.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - auto drvPath = writeDerivation(*store, got, NoRepair, true); + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + + { + auto * checksForAllOutputs_ = std::get_if<0>(&options.outputChecks); + ASSERT_TRUE(checksForAllOutputs_ != nullptr); + auto & checksForAllOutputs = *checksForAllOutputs_; + + EXPECT_EQ( + checksForAllOutputs.allowedReferences, + StringSet{"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"}); + EXPECT_EQ( + checksForAllOutputs.allowedRequisites, + StringSet{"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"}); + EXPECT_EQ( + checksForAllOutputs.disallowedReferences, + StringSet{"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"}); + EXPECT_EQ( + checksForAllOutputs.disallowedRequisites, + StringSet{"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"}); + } + + StringSet systemFeatures{"rainbow", "uid-range"}; + systemFeatures.insert("ca-derivations"); + + EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures); + }); +}; + +TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs_defaults) +{ + this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); ParsedDerivation parsedDrv(drvPath, got); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); @@ -176,25 +280,50 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttr EXPECT_EQ(checksPerOutput.size(), 0); } - EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet()); - EXPECT_EQ(options.canBuildLocally(*store, got), false); - EXPECT_EQ(options.willBuildLocally(*store, got), false); + EXPECT_EQ(options.canBuildLocally(*this->store, got), false); + EXPECT_EQ(options.willBuildLocally(*this->store, got), false); EXPECT_EQ(options.substitutesAllowed(), true); EXPECT_EQ(options.useUidRange(got), false); }); }; -TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttrs) +TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_defaults) { - readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) { - auto got = parseDerivation(*store, std::move(encoded), "foo"); + this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); - auto drvPath = writeDerivation(*store, got, NoRepair, true); + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); ParsedDerivation parsedDrv(drvPath, got); DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); - StringSet systemFeatures{"rainbow", "uid-range"}; + EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{}); + }); +}; + +TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs_defaults) +{ + this->readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + + EXPECT_EQ(options.getRequiredSystemFeatures(got), StringSet{"ca-derivations"}); + }); +}; + +TYPED_TEST(DerivationAdvancedAttrsBothTest, advancedAttributes_structuredAttrs) +{ + this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); EXPECT_TRUE(parsedDrv.hasStructuredAttrs()); @@ -204,14 +333,40 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttr EXPECT_EQ(options.impureEnvVars, StringSet{"UNICORN"}); EXPECT_EQ(options.allowLocalNetworking, true); + { + auto output_ = get(std::get<1>(options.outputChecks), "dev"); + ASSERT_TRUE(output_); + auto & output = *output_; + + EXPECT_EQ(output.maxSize, 789); + EXPECT_EQ(output.maxClosureSize, 5909); + } + + EXPECT_EQ(options.canBuildLocally(*this->store, got), false); + EXPECT_EQ(options.willBuildLocally(*this->store, got), false); + EXPECT_EQ(options.substitutesAllowed(), false); + EXPECT_EQ(options.useUidRange(got), true); + }); +}; + +TEST_F(DerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) +{ + this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + { { auto output_ = get(std::get<1>(options.outputChecks), "out"); ASSERT_TRUE(output_); auto & output = *output_; - EXPECT_EQ(output.allowedReferences, StringSet{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); - EXPECT_EQ(output.allowedRequisites, StringSet{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + EXPECT_EQ(output.allowedReferences, StringSet{"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"}); + EXPECT_EQ(output.allowedRequisites, StringSet{"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"}); } { @@ -219,25 +374,54 @@ TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttr ASSERT_TRUE(output_); auto & output = *output_; - EXPECT_EQ(output.disallowedReferences, StringSet{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); - EXPECT_EQ(output.disallowedRequisites, StringSet{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); - } - - { - auto output_ = get(std::get<1>(options.outputChecks), "dev"); - ASSERT_TRUE(output_); - auto & output = *output_; - - EXPECT_EQ(output.maxSize, 789); - EXPECT_EQ(output.maxClosureSize, 5909); + EXPECT_EQ(output.disallowedReferences, StringSet{"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"}); + EXPECT_EQ( + output.disallowedRequisites, StringSet{"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"}); } } + StringSet systemFeatures{"rainbow", "uid-range"}; + + EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures); + }); +}; + +TEST_F(CaDerivationAdvancedAttrsTest, advancedAttributes_structuredAttrs) +{ + this->readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) { + auto got = parseDerivation(*this->store, std::move(encoded), "foo", this->mockXpSettings); + + auto drvPath = writeDerivation(*this->store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + DerivationOptions options = DerivationOptions::fromParsedDerivation(parsedDrv); + + { + { + auto output_ = get(std::get<1>(options.outputChecks), "out"); + ASSERT_TRUE(output_); + auto & output = *output_; + + EXPECT_EQ(output.allowedReferences, StringSet{"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"}); + EXPECT_EQ(output.allowedRequisites, StringSet{"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"}); + } + + { + auto output_ = get(std::get<1>(options.outputChecks), "bin"); + ASSERT_TRUE(output_); + auto & output = *output_; + + EXPECT_EQ( + output.disallowedReferences, StringSet{"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"}); + EXPECT_EQ( + output.disallowedRequisites, StringSet{"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"}); + } + } + + StringSet systemFeatures{"rainbow", "uid-range"}; + systemFeatures.insert("ca-derivations"); + EXPECT_EQ(options.getRequiredSystemFeatures(got), systemFeatures); - EXPECT_EQ(options.canBuildLocally(*store, got), false); - EXPECT_EQ(options.willBuildLocally(*store, got), false); - EXPECT_EQ(options.substitutesAllowed(), false); - EXPECT_EQ(options.useUidRange(got), true); }); }; diff --git a/src/libstore-tests/machines.cc b/src/libstore-tests/machines.cc index 1d574ceeb..084807130 100644 --- a/src/libstore-tests/machines.cc +++ b/src/libstore-tests/machines.cc @@ -73,6 +73,32 @@ TEST(machines, getMachinesWithSemicolonSeparator) { EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl")))); } +TEST(machines, getMachinesWithCommentsAndSemicolonSeparator) { + auto actual = Machine::parseConfig({}, + "# This is a comment ; this is still that comment\n" + "nix@scratchy.labs.cs.uu.nl ; nix@itchy.labs.cs.uu.nl\n" + "# This is also a comment ; this also is still that comment\n" + "nix@scabby.labs.cs.uu.nl\n"); + EXPECT_THAT(actual, SizeIs(3)); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scabby.labs.cs.uu.nl")))); +} + +TEST(machines, getMachinesWithFunnyWhitespace) { + auto actual = Machine::parseConfig({}, + " # commment ; comment\n" + " nix@scratchy.labs.cs.uu.nl ; nix@itchy.labs.cs.uu.nl \n" + "\n \n" + "\n ;;; \n" + "\n ; ; \n" + "nix@scabby.labs.cs.uu.nl\n\n"); + EXPECT_THAT(actual, SizeIs(3)); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scabby.labs.cs.uu.nl")))); +} + TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) { auto actual = Machine::parseConfig({}, "nix@scratchy.labs.cs.uu.nl i686-linux " diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index 962222f6d..af3a319e9 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -68,7 +68,6 @@ DerivationOptions DerivationOptions::fromParsedDerivation(const ParsedDerivation throw Error("attribute '%s' must be a list of strings", name); res.insert(j->get()); } - checks.disallowedRequisites = res; return res; } return {}; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 360d19afe..fdfdc37b4 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1368,7 +1368,7 @@ Derivation Derivation::fromJSON( for (auto & [outputName, output] : getObject(valueAt(json, "outputs"))) { res.outputs.insert_or_assign( outputName, - DerivationOutput::fromJSON(store, res.name, outputName, output)); + DerivationOutput::fromJSON(store, res.name, outputName, output, xpSettings)); } } catch (Error & e) { e.addTrace({}, "while reading key 'outputs'"); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 055c0739d..fb7c6c7a2 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -22,10 +22,8 @@ #include -#include #include #include -#include #include #include #include @@ -95,7 +93,7 @@ struct curlFileTransfer : public FileTransfer : fileTransfer(fileTransfer) , request(request) , act(*logger, lvlTalkative, actFileTransfer, - request.post ? "" : fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), + fmt("%sing '%s'", request.verb(), request.uri), {request.uri}, request.parentAct) , callback(std::move(callback)) , finalSink([this](std::string_view data) { @@ -272,19 +270,11 @@ struct curlFileTransfer : public FileTransfer return getInterrupted(); } - int silentProgressCallback(curl_off_t dltotal, curl_off_t dlnow) - { - return getInterrupted(); - } - static int progressCallbackWrapper(void * userp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) { - return ((TransferItem *) userp)->progressCallback(dltotal, dlnow); - } - - static int silentProgressCallbackWrapper(void * userp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) - { - return ((TransferItem *) userp)->silentProgressCallback(dltotal, dlnow); + auto & item = *static_cast(userp); + auto isUpload = bool(item.request.data); + return item.progressCallback(isUpload ? ultotal : dltotal, isUpload ? ulnow : dlnow); } static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr) @@ -353,10 +343,7 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper); curl_easy_setopt(req, CURLOPT_HEADERDATA, this); - if (request.post) - curl_easy_setopt(req, CURLOPT_XFERINFOFUNCTION, silentProgressCallbackWrapper); - else - curl_easy_setopt(req, CURLOPT_XFERINFOFUNCTION, progressCallbackWrapper); + curl_easy_setopt(req, CURLOPT_XFERINFOFUNCTION, progressCallbackWrapper); curl_easy_setopt(req, CURLOPT_XFERINFODATA, this); curl_easy_setopt(req, CURLOPT_NOPROGRESS, 0); @@ -449,8 +436,7 @@ struct curlFileTransfer : public FileTransfer if (httpStatus == 304 && result.etag == "") result.etag = request.expectedETag; - if (!request.post) - act.progress(result.bodySize, result.bodySize); + act.progress(result.bodySize, result.bodySize); done = true; callback(std::move(result)); } @@ -539,6 +525,8 @@ struct curlFileTransfer : public FileTransfer warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); else warn("%s; retrying in %d ms", exc.what(), ms); + decompressionSink.reset(); + errorSink.reset(); embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); fileTransfer.enqueueItem(shared_from_this()); } @@ -791,10 +779,6 @@ struct curlFileTransfer : public FileTransfer S3Helper s3Helper(profile, region, scheme, endpoint); - Activity act(*logger, lvlTalkative, actFileTransfer, - fmt("downloading '%s'", request.uri), - {request.uri}, request.parentAct); - // FIXME: implement ETag auto s3Res = s3Helper.getObject(bucketName, key); FileTransferResult res; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 76ebd16ba..e4c1f8819 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -280,21 +280,21 @@ template<> void BaseSetting::convertToArg(Args & args, const std::s .aliases = aliases, .description = "Enable sandboxing.", .category = category, - .handler = {[this]() { override(smEnabled); }} + .handler = {[this]() { override(smEnabled); }}, }); args.addFlag({ .longName = "no-" + name, .aliases = aliases, .description = "Disable sandboxing.", .category = category, - .handler = {[this]() { override(smDisabled); }} + .handler = {[this]() { override(smDisabled); }}, }); args.addFlag({ .longName = "relaxed-" + name, .aliases = aliases, .description = "Enable sandboxing, but allow builds to disable it.", .category = category, - .handler = {[this]() { override(smRelaxed); }} + .handler = {[this]() { override(smRelaxed); }}, }); } diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 217c52d77..f87f68e7f 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -77,7 +77,7 @@ struct FileTransferRequest FileTransferRequest(std::string_view uri) : uri(uri), parentAct(getCurActivity()) { } - std::string verb() + std::string verb() const { return data ? "upload" : "download"; } diff --git a/src/libstore/linux/include/nix/store/fchmodat2-compat.hh b/src/libstore/linux/fchmodat2-compat.hh similarity index 100% rename from src/libstore/linux/include/nix/store/fchmodat2-compat.hh rename to src/libstore/linux/fchmodat2-compat.hh diff --git a/src/libstore/linux/include/nix/store/meson.build b/src/libstore/linux/include/nix/store/meson.build index fd05fcaea..a664aefa9 100644 --- a/src/libstore/linux/include/nix/store/meson.build +++ b/src/libstore/linux/include/nix/store/meson.build @@ -1,6 +1,5 @@ include_dirs += include_directories('../..') headers += files( - 'fchmodat2-compat.hh', 'personality.hh', ) diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 7c077239d..d98d06651 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -105,28 +105,31 @@ ref Machine::openStore() const static std::vector expandBuilderLines(const std::string & builders) { std::vector result; - for (auto line : tokenizeString>(builders, "\n;")) { - trim(line); + for (auto line : tokenizeString>(builders, "\n")) { line.erase(std::find(line.begin(), line.end(), '#'), line.end()); - if (line.empty()) continue; + for (auto entry : tokenizeString>(line, ";")) { + entry = trim(entry); - if (line[0] == '@') { - const std::string path = trim(std::string(line, 1)); - std::string text; - try { - text = readFile(path); - } catch (const SysError & e) { - if (e.errNo != ENOENT) - throw; - debug("cannot find machines file '%s'", path); + if (entry.empty()) { + // skip blank entries + } else if (entry[0] == '@') { + const std::string path = trim(std::string_view{entry}.substr(1)); + std::string text; + try { + text = readFile(path); + } catch (const SysError & e) { + if (e.errNo != ENOENT) + throw; + debug("cannot find machines file '%s'", path); + continue; + } + + const auto entrys = expandBuilderLines(text); + result.insert(end(result), begin(entrys), end(entrys)); + } else { + result.emplace_back(entry); } - - const auto lines = expandBuilderLines(text); - result.insert(end(result), begin(lines), end(lines)); - continue; } - - result.emplace_back(line); } return result; } diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 87f5feb45..f9e583307 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -160,7 +160,10 @@ ref S3Helper::makeConfig( S3Helper::FileTransferResult S3Helper::getObject( const std::string & bucketName, const std::string & key) { - debug("fetching 's3://%s/%s'...", bucketName, key); + std::string uri = "s3://" + bucketName + "/" + key; + Activity act(*logger, lvlTalkative, actFileTransfer, + fmt("downloading '%s'", uri), + Logger::Fields{uri}, getCurActivity()); auto request = Aws::S3::Model::GetObjectRequest() @@ -171,6 +174,22 @@ S3Helper::FileTransferResult S3Helper::getObject( return Aws::New("STRINGSTREAM"); }); + size_t bytesDone = 0; + size_t bytesExpected = 0; + request.SetDataReceivedEventHandler([&](const Aws::Http::HttpRequest * req, Aws::Http::HttpResponse * resp, long long l) { + if (!bytesExpected && resp->HasHeader("Content-Length")) { + if (auto length = string2Int(resp->GetHeader("Content-Length"))) { + bytesExpected = *length; + } + } + bytesDone += l; + act.progress(bytesDone, bytesExpected); + }); + + request.SetContinueRequestHandler([](const Aws::Http::HttpRequest*) { + return !isInterrupted(); + }); + FileTransferResult res; auto now1 = std::chrono::steady_clock::now(); @@ -180,6 +199,8 @@ S3Helper::FileTransferResult S3Helper::getObject( auto result = checkAws(fmt("AWS error fetching '%s'", key), client->GetObject(request)); + act.progress(result.GetContentLength(), result.GetContentLength()); + res.data = decompress(result.GetContentEncoding(), dynamic_cast(result.GetBody()).str()); @@ -307,11 +328,35 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual std::shared_ptr transferManager; std::once_flag transferManagerCreated; + struct AsyncContext : public Aws::Client::AsyncCallerContext + { + mutable std::mutex mutex; + mutable std::condition_variable cv; + const Activity & act; + + void notify() const + { + cv.notify_one(); + } + + void wait() const + { + std::unique_lock lk(mutex); + cv.wait(lk); + } + + AsyncContext(const Activity & act) : act(act) {} + }; + void uploadFile(const std::string & path, std::shared_ptr> istream, const std::string & mimeType, const std::string & contentEncoding) { + std::string uri = "s3://" + bucketName + "/" + path; + Activity act(*logger, lvlTalkative, actFileTransfer, + fmt("uploading '%s'", uri), + Logger::Fields{uri}, getCurActivity()); istream->seekg(0, istream->end); auto size = istream->tellg(); istream->seekg(0, istream->beg); @@ -330,16 +375,25 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual transferConfig.bufferSize = bufferSize; transferConfig.uploadProgressCallback = - [](const TransferManager *transferManager, - const std::shared_ptr - &transferHandle) + [](const TransferManager * transferManager, + const std::shared_ptr & transferHandle) { - //FIXME: find a way to properly abort the multipart upload. - //checkInterrupt(); - debug("upload progress ('%s'): '%d' of '%d' bytes", - transferHandle->GetKey(), - transferHandle->GetBytesTransferred(), - transferHandle->GetBytesTotalSize()); + auto context = std::dynamic_pointer_cast(transferHandle->GetContext()); + size_t bytesDone = transferHandle->GetBytesTransferred(); + size_t bytesTotal = transferHandle->GetBytesTotalSize(); + try { + checkInterrupt(); + context->act.progress(bytesDone, bytesTotal); + } catch (...) { + context->notify(); + } + }; + transferConfig.transferStatusUpdatedCallback = + [](const TransferManager * transferManager, + const std::shared_ptr & transferHandle) + { + auto context = std::dynamic_pointer_cast(transferHandle->GetContext()); + context->notify(); }; transferManager = TransferManager::Create(transferConfig); @@ -353,29 +407,51 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual if (contentEncoding != "") throw Error("setting a content encoding is not supported with S3 multi-part uploads"); + auto context = std::make_shared(act); std::shared_ptr transferHandle = transferManager->UploadFile( istream, bucketName, path, mimeType, Aws::Map(), - nullptr /*, contentEncoding */); + context /*, contentEncoding */); - transferHandle->WaitUntilFinished(); + TransferStatus status = transferHandle->GetStatus(); + while (status == TransferStatus::IN_PROGRESS || status == TransferStatus::NOT_STARTED) { + if (!isInterrupted()) { + context->wait(); + } else { + transferHandle->Cancel(); + transferHandle->WaitUntilFinished(); + } + status = transferHandle->GetStatus(); + } + act.progress(transferHandle->GetBytesTransferred(), transferHandle->GetBytesTotalSize()); - if (transferHandle->GetStatus() == TransferStatus::FAILED) + if (status == TransferStatus::FAILED) throw Error("AWS error: failed to upload 's3://%s/%s': %s", bucketName, path, transferHandle->GetLastError().GetMessage()); - if (transferHandle->GetStatus() != TransferStatus::COMPLETED) + if (status != TransferStatus::COMPLETED) throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state", bucketName, path); } else { + act.progress(0, size); auto request = Aws::S3::Model::PutObjectRequest() .WithBucket(bucketName) .WithKey(path); + size_t bytesSent = 0; + request.SetDataSentEventHandler([&](const Aws::Http::HttpRequest * req, long long l) { + bytesSent += l; + act.progress(bytesSent, size); + }); + + request.SetContinueRequestHandler([](const Aws::Http::HttpRequest*) { + return !isInterrupted(); + }); + request.SetContentType(mimeType); if (contentEncoding != "") @@ -385,6 +461,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual auto result = checkAws(fmt("AWS error uploading '%s'", path), s3Helper.client->PutObject(request)); + + act.progress(size, size); } auto now2 = std::chrono::steady_clock::now(); diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index ce0664b1c..3ba1e823f 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -42,7 +42,7 @@ /* Includes required for chroot support. */ #ifdef __linux__ -# include "nix/store/fchmodat2-compat.hh" +# include "linux/fchmodat2-compat.hh" # include # include # include diff --git a/src/libutil-test-support/include/nix/util/tests/tracing-file-system-object-sink.hh b/src/libutil-test-support/include/nix/util/tests/tracing-file-system-object-sink.hh deleted file mode 100644 index d721c13af..000000000 --- a/src/libutil-test-support/include/nix/util/tests/tracing-file-system-object-sink.hh +++ /dev/null @@ -1,41 +0,0 @@ -#pragma once -#include "nix/util/fs-sink.hh" - -namespace nix::test { - -/** - * A `FileSystemObjectSink` that traces calls, writing to stderr. - */ -class TracingFileSystemObjectSink : public virtual FileSystemObjectSink -{ - FileSystemObjectSink & sink; -public: - TracingFileSystemObjectSink(FileSystemObjectSink & sink) - : sink(sink) - { - } - - void createDirectory(const CanonPath & path) override; - - void createRegularFile(const CanonPath & path, std::function fn) override; - - void createSymlink(const CanonPath & path, const std::string & target) override; -}; - -/** - * A `ExtendedFileSystemObjectSink` that traces calls, writing to stderr. - */ -class TracingExtendedFileSystemObjectSink : public TracingFileSystemObjectSink, public ExtendedFileSystemObjectSink -{ - ExtendedFileSystemObjectSink & sink; -public: - TracingExtendedFileSystemObjectSink(ExtendedFileSystemObjectSink & sink) - : TracingFileSystemObjectSink(sink) - , sink(sink) - { - } - - void createHardlink(const CanonPath & path, const CanonPath & target) override; -}; - -} diff --git a/src/libutil-test-support/tracing-file-system-object-sink.cc b/src/libutil-test-support/tracing-file-system-object-sink.cc deleted file mode 100644 index 52b081fb8..000000000 --- a/src/libutil-test-support/tracing-file-system-object-sink.cc +++ /dev/null @@ -1,34 +0,0 @@ -#include -#include "nix/tracing-file-system-object-sink.hh" - -namespace nix::test { - -void TracingFileSystemObjectSink::createDirectory(const CanonPath & path) -{ - std::cerr << "createDirectory(" << path << ")\n"; - sink.createDirectory(path); -} - -void TracingFileSystemObjectSink::createRegularFile( - const CanonPath & path, std::function fn) -{ - std::cerr << "createRegularFile(" << path << ")\n"; - sink.createRegularFile(path, [&](CreateRegularFileSink & crf) { - // We could wrap this and trace about the chunks of data and such - fn(crf); - }); -} - -void TracingFileSystemObjectSink::createSymlink(const CanonPath & path, const std::string & target) -{ - std::cerr << "createSymlink(" << path << ", target: " << target << ")\n"; - sink.createSymlink(path, target); -} - -void TracingExtendedFileSystemObjectSink::createHardlink(const CanonPath & path, const CanonPath & target) -{ - std::cerr << "createHardlink(" << path << ", target: " << target << ")\n"; - sink.createHardlink(path, target); -} - -} // namespace nix::test diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index 8f9c18eed..f2552550d 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -59,6 +59,7 @@ sources = files( 'json-utils.cc', 'logging.cc', 'lru-cache.cc', + 'monitorfdhup.cc', 'nix_api_util.cc', 'pool.cc', 'position.cc', diff --git a/src/libutil-tests/monitorfdhup.cc b/src/libutil-tests/monitorfdhup.cc index 01ecb92d9..f9da4022d 100644 --- a/src/libutil-tests/monitorfdhup.cc +++ b/src/libutil-tests/monitorfdhup.cc @@ -1,5 +1,5 @@ -#include "util.hh" -#include "monitor-fd.hh" +#include "nix/util/util.hh" +#include "nix/util/monitor-fd.hh" #include #include diff --git a/src/libutil/include/nix/util/signals.hh b/src/libutil/include/nix/util/signals.hh index 45130a90c..5a2ba8e75 100644 --- a/src/libutil/include/nix/util/signals.hh +++ b/src/libutil/include/nix/util/signals.hh @@ -26,6 +26,11 @@ static inline bool getInterrupted(); */ void setInterruptThrown(); +/** + * @note Does nothing on Windows + */ +static inline bool isInterrupted(); + /** * @note Does nothing on Windows */ diff --git a/src/libutil/unix/include/nix/util/signals-impl.hh b/src/libutil/unix/include/nix/util/signals-impl.hh index ffa967344..7397744b2 100644 --- a/src/libutil/unix/include/nix/util/signals-impl.hh +++ b/src/libutil/unix/include/nix/util/signals-impl.hh @@ -85,17 +85,22 @@ static inline bool getInterrupted() return unix::_isInterrupted; } +static inline bool isInterrupted() +{ + using namespace unix; + return _isInterrupted || (interruptCheck && interruptCheck()); +} + /** * Throw `Interrupted` exception if the process has been interrupted. * * Call this in long-running loops and between slow operations to terminate * them as needed. */ -void inline checkInterrupt() +inline void checkInterrupt() { - using namespace unix; - if (_isInterrupted || (interruptCheck && interruptCheck())) - _interrupted(); + if (isInterrupted()) + unix::_interrupted(); } /** diff --git a/src/libutil/windows/include/nix/util/signals-impl.hh b/src/libutil/windows/include/nix/util/signals-impl.hh index 043f39100..f716ffd1a 100644 --- a/src/libutil/windows/include/nix/util/signals-impl.hh +++ b/src/libutil/windows/include/nix/util/signals-impl.hh @@ -22,7 +22,12 @@ inline void setInterruptThrown() /* Do nothing for now */ } -void inline checkInterrupt() +static inline bool isInterrupted() +{ + /* Do nothing for now */ +} + +inline void checkInterrupt() { /* Do nothing for now */ } diff --git a/src/nix/build.cc b/src/nix/build.cc index 7cd3c7fbe..8db831240 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -55,7 +55,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile .description = "Use *path* as prefix for the symlinks to the build results. It defaults to `result`.", .labels = {"path"}, .handler = {&outLink}, - .completer = completePath + .completer = completePath, }); addFlag({ diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 30b3003e7..c334469b5 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -24,7 +24,7 @@ struct CmdBundle : InstallableValueCommand .handler = {&bundler}, .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { completeFlakeRef(completions, getStore(), prefix); - }} + }}, }); addFlag({ @@ -33,7 +33,7 @@ struct CmdBundle : InstallableValueCommand .description = "Override the name of the symlink to the build result. It defaults to the base name of the app.", .labels = {"path"}, .handler = {&outLink}, - .completer = completePath + .completer = completePath, }); } diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 0702215fd..013f2a7e3 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -21,7 +21,7 @@ struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile .description = "Create symlinks prefixed with *path* to the top-level store paths fetched from the source store.", .labels = {"path"}, .handler = {&outLink}, - .completer = completePath + .completer = completePath, }); addFlag({ diff --git a/src/nix/derivation-show.cc b/src/nix/derivation-show.cc index 050144ccf..86755c3e8 100644 --- a/src/nix/derivation-show.cc +++ b/src/nix/derivation-show.cc @@ -21,7 +21,7 @@ struct CmdShowDerivation : InstallablesCommand .longName = "recursive", .shortName = 'r', .description = "Include the dependencies of the specified derivations.", - .handler = {&recursive, true} + .handler = {&recursive, true}, }); } diff --git a/src/nix/develop.cc b/src/nix/develop.cc index e88134a78..00572697a 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -334,7 +334,7 @@ struct Common : InstallableCommand, MixProfile .labels = {"installable", "outputs-dir"}, .handler = {[&](std::string installable, std::string outputsDir) { redirects.push_back({installable, outputsDir}); - }} + }}, }); } @@ -524,7 +524,7 @@ struct CmdDevelop : Common, MixEnvironment .handler = {[&](std::vector ss) { if (ss.empty()) throw UsageError("--command requires at least one argument"); command = ss; - }} + }}, }); addFlag({ diff --git a/src/nix/env.cc b/src/nix/env.cc index 4b00dbc7c..f6b12f21c 100644 --- a/src/nix/env.cc +++ b/src/nix/env.cc @@ -38,16 +38,17 @@ struct CmdShell : InstallablesCommand, MixEnvironment CmdShell() { - addFlag( - {.longName = "command", - .shortName = 'c', - .description = "Command and arguments to be executed, defaulting to `$SHELL`", - .labels = {"command", "args"}, - .handler = {[&](std::vector ss) { - if (ss.empty()) - throw UsageError("--command requires at least one argument"); - command = ss; - }}}); + addFlag({ + .longName = "command", + .shortName = 'c', + .description = "Command and arguments to be executed, defaulting to `$SHELL`", + .labels = {"command", "args"}, + .handler = {[&](std::vector ss) { + if (ss.empty()) + throw UsageError("--command requires at least one argument"); + command = ss; + }}, + }); } std::string description() override diff --git a/src/nix/flake.cc b/src/nix/flake.cc index f429f1c6e..6df025e35 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -90,7 +90,7 @@ public: .handler={&flakeUrl}, .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { completeFlakeRef(completions, getStore(), prefix); - }} + }}, }); expectArgs({ .label="inputs", @@ -111,7 +111,7 @@ public: }}, .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { completeFlakeInputAttrPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); - }} + }}, }); /* Remove flags that don't make sense. */ @@ -336,12 +336,12 @@ struct CmdFlakeCheck : FlakeCommand addFlag({ .longName = "no-build", .description = "Do not build checks.", - .handler = {&build, false} + .handler = {&build, false}, }); addFlag({ .longName = "all-systems", .description = "Check the outputs for all systems.", - .handler = {&checkAllSystems, true} + .handler = {&checkAllSystems, true}, }); } @@ -874,7 +874,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand defaultTemplateAttrPathsPrefixes, defaultTemplateAttrPaths, prefix); - }} + }}, }); } @@ -1034,7 +1034,7 @@ struct CmdFlakeClone : FlakeCommand .shortName = 'f', .description = "Clone the flake to path *dest*.", .labels = {"path"}, - .handler = {&destDir} + .handler = {&destDir}, }); } @@ -1057,7 +1057,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun .longName = "to", .description = "URI of the destination Nix store", .labels = {"store-uri"}, - .handler = {&dstUri} + .handler = {&dstUri}, }); } @@ -1137,12 +1137,12 @@ struct CmdFlakeShow : FlakeCommand, MixJSON addFlag({ .longName = "legacy", .description = "Show the contents of the `legacyPackages` output.", - .handler = {&showLegacy, true} + .handler = {&showLegacy, true}, }); addFlag({ .longName = "all-systems", .description = "Show the contents of outputs for all systems.", - .handler = {&showAllSystems, true} + .handler = {&showAllSystems, true}, }); } @@ -1443,7 +1443,7 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON .description = "Create symlink named *path* to the resulting store path.", .labels = {"path"}, .handler = {&outLink}, - .completer = completePath + .completer = completePath, }); } diff --git a/src/nix/meson.build b/src/nix/meson.build index 3cb45f1f5..901021330 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -7,7 +7,7 @@ project('nix', 'cpp', 'errorlogs=true', # Please print logs for tests that fail 'localstatedir=/nix/var', ], - meson_version : '>= 1.1', + meson_version : '>= 1.4', license : 'LGPL-2.1-or-later', ) diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 397134b03..4495a1489 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -275,7 +275,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON .longName = "name", .description = "Override the name component of the resulting store path. It defaults to the base name of *url*.", .labels = {"name"}, - .handler = {&name} + .handler = {&name}, }); addFlag({ @@ -284,7 +284,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON .labels = {"hash"}, .handler = {[&](std::string s) { expectedHash = Hash::parseAny(s, hashAlgo); - }} + }}, }); addFlag(flag::hashAlgo("hash-type", &hashAlgo)); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 87d0e1edb..9ef54a414 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -104,7 +104,7 @@ struct CmdSign : StorePathsCommand .description = "File containing the secret signing key.", .labels = {"file"}, .handler = {&secretKeyFile}, - .completer = completePath + .completer = completePath, }); } diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index f71a56bc7..fae960c90 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -16,7 +16,7 @@ struct CmdStoreDelete : StorePathsCommand addFlag({ .longName = "ignore-liveness", .description = "Do not check whether the paths are reachable from a root.", - .handler = {&options.ignoreLiveness, true} + .handler = {&options.ignoreLiveness, true}, }); } diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc index e6a303874..c71e89233 100644 --- a/src/nix/store-gc.cc +++ b/src/nix/store-gc.cc @@ -17,7 +17,7 @@ struct CmdStoreGC : StoreCommand, MixDryRun .longName = "max", .description = "Stop after freeing *n* bytes of disk space.", .labels = {"n"}, - .handler = {&options.maxFreed} + .handler = {&options.maxFreed}, }); } diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index c0a6e6827..648241104 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -23,14 +23,14 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand .shortName = 'p', .description = "The path to the Nix profile to upgrade.", .labels = {"profile-dir"}, - .handler = {&profileDir} + .handler = {&profileDir}, }); addFlag({ .longName = "nix-store-paths-url", .description = "The URL of the file that contains the store paths of the latest Nix release.", .labels = {"url"}, - .handler = {&(std::string&) settings.upgradeNixStorePathUrl} + .handler = {&(std::string&) settings.upgradeNixStorePathUrl}, }); } diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 734387ee7..ff81d78b6 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -37,7 +37,7 @@ struct CmdVerify : StorePathsCommand .shortName = 's', .description = "Use signatures from the specified store.", .labels = {"store-uri"}, - .handler = {[&](std::string s) { substituterUris.push_back(s); }} + .handler = {[&](std::string s) { substituterUris.push_back(s); }}, }); addFlag({ @@ -45,7 +45,7 @@ struct CmdVerify : StorePathsCommand .shortName = 'n', .description = "Require that each path is signed by at least *n* different keys.", .labels = {"n"}, - .handler = {&sigsNeeded} + .handler = {&sigsNeeded}, }); } diff --git a/tests/functional/build-remote-content-addressed-floating.sh b/tests/functional/build-remote-content-addressed-floating.sh index 33d667f92..370915905 100755 --- a/tests/functional/build-remote-content-addressed-floating.sh +++ b/tests/functional/build-remote-content-addressed-floating.sh @@ -6,6 +6,6 @@ file=build-hook-ca-floating.nix enableFeatures "ca-derivations" -CONTENT_ADDRESSED=true +NIX_TESTS_CA_BY_DEFAULT=true source build-remote.sh diff --git a/tests/functional/build-remote.sh b/tests/functional/build-remote.sh index 3231341cb..62cc85888 100644 --- a/tests/functional/build-remote.sh +++ b/tests/functional/build-remote.sh @@ -13,7 +13,7 @@ unset NIX_STATE_DIR function join_by { local d=$1; shift; echo -n "$1"; shift; printf "%s" "${@/#/$d}"; } EXTRA_SYSTEM_FEATURES=() -if [[ -n "${CONTENT_ADDRESSED-}" ]]; then +if [[ -n "${NIX_TESTS_CA_BY_DEFAULT-}" ]]; then EXTRA_SYSTEM_FEATURES=("ca-derivations") fi diff --git a/tests/functional/ca/derivation-advanced-attributes.sh b/tests/functional/ca/derivation-advanced-attributes.sh new file mode 100755 index 000000000..b70463e5c --- /dev/null +++ b/tests/functional/ca/derivation-advanced-attributes.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +export NIX_TESTS_CA_BY_DEFAULT=1 + +cd .. +source derivation-advanced-attributes.sh diff --git a/tests/functional/ca/meson.build b/tests/functional/ca/meson.build index 7a7fcc5cf..a4611ca42 100644 --- a/tests/functional/ca/meson.build +++ b/tests/functional/ca/meson.build @@ -8,10 +8,11 @@ suites += { 'name': 'ca', 'deps': [], 'tests': [ + 'build-cache.sh', 'build-with-garbage-path.sh', 'build.sh', - 'build-cache.sh', 'concurrent-builds.sh', + 'derivation-advanced-attributes.sh', 'derivation-json.sh', 'duplicate-realisation-in-closure.sh', 'eval-store.sh', diff --git a/tests/functional/ca/nix-shell.sh b/tests/functional/ca/nix-shell.sh index d1fbe54d1..7b30b2ac8 100755 --- a/tests/functional/ca/nix-shell.sh +++ b/tests/functional/ca/nix-shell.sh @@ -2,6 +2,6 @@ source common.sh -CONTENT_ADDRESSED=true +NIX_TESTS_CA_BY_DEFAULT=true cd .. source ./nix-shell.sh diff --git a/tests/functional/derivation-advanced-attributes.sh b/tests/functional/derivation-advanced-attributes.sh index 6707b345c..a7530e11c 100755 --- a/tests/functional/derivation-advanced-attributes.sh +++ b/tests/functional/derivation-advanced-attributes.sh @@ -12,11 +12,19 @@ badExitCode=0 store="$TEST_ROOT/store" +if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then + drvDir=ia + flags=(--arg contentAddress false) +else + drvDir=ca + flags=(--arg contentAddress true --extra-experimental-features ca-derivations) +fi + for nixFile in derivation/*.nix; do - drvPath=$(env -u NIX_STORE nix-instantiate --store "$store" --pure-eval --expr "$(< "$nixFile")") + drvPath=$(env -u NIX_STORE nix-instantiate --store "$store" --pure-eval "${flags[@]}" --expr "$(< "$nixFile")") testName=$(basename "$nixFile" .nix) got="${store}${drvPath}" - expected="derivation/$testName.drv" + expected="derivation/${drvDir}/${testName}.drv" diffAndAcceptInner "$testName" "$got" "$expected" done diff --git a/tests/functional/derivation/advanced-attributes-defaults.nix b/tests/functional/derivation/advanced-attributes-defaults.nix index d466003b0..51f359cf0 100644 --- a/tests/functional/derivation/advanced-attributes-defaults.nix +++ b/tests/functional/derivation/advanced-attributes-defaults.nix @@ -1,6 +1,24 @@ -derivation { - name = "advanced-attributes-defaults"; +{ contentAddress }: + +let + caArgs = + if contentAddress then + { + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } + else + { }; + + derivation' = args: derivation (caArgs // args); + system = "my-system"; + +in +derivation' { + inherit system; + name = "advanced-attributes-defaults"; builder = "/bin/bash"; args = [ "-c" diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix index 3c6ad4900..ec51f0e28 100644 --- a/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix +++ b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix @@ -1,6 +1,24 @@ -derivation { - name = "advanced-attributes-structured-attrs-defaults"; +{ contentAddress }: + +let + caArgs = + if contentAddress then + { + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } + else + { }; + + derivation' = args: derivation (caArgs // args); + system = "my-system"; + +in +derivation' { + inherit system; + name = "advanced-attributes-structured-attrs-defaults"; builder = "/bin/bash"; args = [ "-c" diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs.drv b/tests/functional/derivation/advanced-attributes-structured-attrs.drv deleted file mode 100644 index e47a41ad5..000000000 --- a/tests/functional/derivation/advanced-attributes-structured-attrs.drv +++ /dev/null @@ -1 +0,0 @@ -Derive([("bin","/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin","",""),("dev","/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev","",""),("out","/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs","","")],[("/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv",["out"]),("/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv",["out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"],\"disallowedRequisites\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"allowedRequisites\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}"),("bin","/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin"),("dev","/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev"),("out","/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs.nix b/tests/functional/derivation/advanced-attributes-structured-attrs.nix index 4c596be45..27d9e7cf9 100644 --- a/tests/functional/derivation/advanced-attributes-structured-attrs.nix +++ b/tests/functional/derivation/advanced-attributes-structured-attrs.nix @@ -1,6 +1,21 @@ +{ contentAddress }: + let + caArgs = + if contentAddress then + { + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } + else + { }; + + derivation' = args: derivation (caArgs // args); + system = "my-system"; - foo = derivation { + + foo = derivation' { inherit system; name = "foo"; builder = "/bin/bash"; @@ -8,8 +23,13 @@ let "-c" "echo foo > $out" ]; + outputs = [ + "out" + "dev" + ]; }; - bar = derivation { + + bar = derivation' { inherit system; name = "bar"; builder = "/bin/bash"; @@ -17,9 +37,14 @@ let "-c" "echo bar > $out" ]; + outputs = [ + "out" + "dev" + ]; }; + in -derivation { +derivation' { inherit system; name = "advanced-attributes-structured-attrs"; builder = "/bin/bash"; @@ -41,11 +66,11 @@ derivation { outputChecks = { out = { allowedReferences = [ foo ]; - allowedRequisites = [ foo ]; + allowedRequisites = [ foo.dev ]; }; bin = { disallowedReferences = [ bar ]; - disallowedRequisites = [ bar ]; + disallowedRequisites = [ bar.dev ]; }; dev = { maxSize = 789; diff --git a/tests/functional/derivation/advanced-attributes.drv b/tests/functional/derivation/advanced-attributes.drv deleted file mode 100644 index ec3112ab2..000000000 --- a/tests/functional/derivation/advanced-attributes.drv +++ /dev/null @@ -1 +0,0 @@ -Derive([("out","/nix/store/33a6fdmn8q9ih9d7npbnrxn2q56a4l8q-advanced-attributes","","")],[("/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv",["out"]),("/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv",["out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__darwinAllowLocalNetworking","1"),("__impureHostDeps","/usr/bin/ditto"),("__noChroot","1"),("__sandboxProfile","sandcastle"),("allowSubstitutes",""),("allowedReferences","/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"),("allowedRequisites","/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"),("builder","/bin/bash"),("disallowedReferences","/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"),("disallowedRequisites","/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"),("impureEnvVars","UNICORN"),("name","advanced-attributes"),("out","/nix/store/33a6fdmn8q9ih9d7npbnrxn2q56a4l8q-advanced-attributes"),("preferLocalBuild","1"),("requiredSystemFeatures","rainbow uid-range"),("system","my-system")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes.nix b/tests/functional/derivation/advanced-attributes.nix index 7f365ce65..e988e0a70 100644 --- a/tests/functional/derivation/advanced-attributes.nix +++ b/tests/functional/derivation/advanced-attributes.nix @@ -1,6 +1,21 @@ +{ contentAddress }: + let + caArgs = + if contentAddress then + { + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + } + else + { }; + + derivation' = args: derivation (caArgs // args); + system = "my-system"; - foo = derivation { + + foo = derivation' { inherit system; name = "foo"; builder = "/bin/bash"; @@ -8,8 +23,13 @@ let "-c" "echo foo > $out" ]; + outputs = [ + "out" + "dev" + ]; }; - bar = derivation { + + bar = derivation' { inherit system; name = "bar"; builder = "/bin/bash"; @@ -17,9 +37,14 @@ let "-c" "echo bar > $out" ]; + outputs = [ + "out" + "dev" + ]; }; + in -derivation { +derivation' { inherit system; name = "advanced-attributes"; builder = "/bin/bash"; @@ -33,9 +58,9 @@ derivation { impureEnvVars = [ "UNICORN" ]; __darwinAllowLocalNetworking = true; allowedReferences = [ foo ]; - allowedRequisites = [ foo ]; + allowedRequisites = [ foo.dev ]; disallowedReferences = [ bar ]; - disallowedRequisites = [ bar ]; + disallowedRequisites = [ bar.dev ]; requiredSystemFeatures = [ "rainbow" "uid-range" diff --git a/tests/functional/derivation/ca/advanced-attributes-defaults.drv b/tests/functional/derivation/ca/advanced-attributes-defaults.drv new file mode 100644 index 000000000..2c8160963 --- /dev/null +++ b/tests/functional/derivation/ca/advanced-attributes-defaults.drv @@ -0,0 +1 @@ +Derive([("out","","r:sha256","")],[],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("builder","/bin/bash"),("name","advanced-attributes-defaults"),("out","/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"),("outputHashAlgo","sha256"),("outputHashMode","recursive"),("system","my-system")]) \ No newline at end of file diff --git a/tests/functional/derivation/ca/advanced-attributes-structured-attrs-defaults.drv b/tests/functional/derivation/ca/advanced-attributes-structured-attrs-defaults.drv new file mode 100644 index 000000000..bf56e05d6 --- /dev/null +++ b/tests/functional/derivation/ca/advanced-attributes-structured-attrs-defaults.drv @@ -0,0 +1 @@ +Derive([("dev","","r:sha256",""),("out","","r:sha256","")],[],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"builder\":\"/bin/bash\",\"name\":\"advanced-attributes-structured-attrs-defaults\",\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"dev\"],\"system\":\"my-system\"}"),("dev","/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz"),("out","/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9")]) \ No newline at end of file diff --git a/tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv b/tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv new file mode 100644 index 000000000..a81e74d41 --- /dev/null +++ b/tests/functional/derivation/ca/advanced-attributes-structured-attrs.drv @@ -0,0 +1 @@ +Derive([("bin","","r:sha256",""),("dev","","r:sha256",""),("out","","r:sha256","")],[("/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",["dev","out"]),("/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",["dev","out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g\"],\"disallowedRequisites\":[\"/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9\"],\"allowedRequisites\":[\"/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z\"]}},\"outputHashAlgo\":\"sha256\",\"outputHashMode\":\"recursive\",\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}"),("bin","/04f3da1kmbr67m3gzxikmsl4vjz5zf777sv6m14ahv22r65aac9m"),("dev","/02qcpld1y6xhs5gz9bchpxaw0xdhmsp5dv88lh25r2ss44kh8dxz"),("out","/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9")]) \ No newline at end of file diff --git a/tests/functional/derivation/ca/advanced-attributes.drv b/tests/functional/derivation/ca/advanced-attributes.drv new file mode 100644 index 000000000..dded6c620 --- /dev/null +++ b/tests/functional/derivation/ca/advanced-attributes.drv @@ -0,0 +1 @@ +Derive([("out","","r:sha256","")],[("/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv",["dev","out"]),("/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",["dev","out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__darwinAllowLocalNetworking","1"),("__impureHostDeps","/usr/bin/ditto"),("__noChroot","1"),("__sandboxProfile","sandcastle"),("allowSubstitutes",""),("allowedReferences","/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"),("allowedRequisites","/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"),("builder","/bin/bash"),("disallowedReferences","/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"),("disallowedRequisites","/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"),("impureEnvVars","UNICORN"),("name","advanced-attributes"),("out","/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9"),("outputHashAlgo","sha256"),("outputHashMode","recursive"),("preferLocalBuild","1"),("requiredSystemFeatures","rainbow uid-range"),("system","my-system")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes-defaults.drv b/tests/functional/derivation/ia/advanced-attributes-defaults.drv similarity index 100% rename from tests/functional/derivation/advanced-attributes-defaults.drv rename to tests/functional/derivation/ia/advanced-attributes-defaults.drv diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv b/tests/functional/derivation/ia/advanced-attributes-structured-attrs-defaults.drv similarity index 100% rename from tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv rename to tests/functional/derivation/ia/advanced-attributes-structured-attrs-defaults.drv diff --git a/tests/functional/derivation/ia/advanced-attributes-structured-attrs.drv b/tests/functional/derivation/ia/advanced-attributes-structured-attrs.drv new file mode 100644 index 000000000..1560bca66 --- /dev/null +++ b/tests/functional/derivation/ia/advanced-attributes-structured-attrs.drv @@ -0,0 +1 @@ +Derive([("bin","/nix/store/qjjj3zrlimpjbkk686m052b3ks9iz2sl-advanced-attributes-structured-attrs-bin","",""),("dev","/nix/store/lpz5grl48v93pdadavyg5is1rqvfdipf-advanced-attributes-structured-attrs-dev","",""),("out","/nix/store/nzvz1bmh1g89a5dkpqcqan0av7q3hgv3-advanced-attributes-structured-attrs","","")],[("/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv",["dev","out"]),("/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv",["dev","out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar\"],\"disallowedRequisites\":[\"/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo\"],\"allowedRequisites\":[\"/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}"),("bin","/nix/store/qjjj3zrlimpjbkk686m052b3ks9iz2sl-advanced-attributes-structured-attrs-bin"),("dev","/nix/store/lpz5grl48v93pdadavyg5is1rqvfdipf-advanced-attributes-structured-attrs-dev"),("out","/nix/store/nzvz1bmh1g89a5dkpqcqan0av7q3hgv3-advanced-attributes-structured-attrs")]) \ No newline at end of file diff --git a/tests/functional/derivation/ia/advanced-attributes.drv b/tests/functional/derivation/ia/advanced-attributes.drv new file mode 100644 index 000000000..2c5d5a692 --- /dev/null +++ b/tests/functional/derivation/ia/advanced-attributes.drv @@ -0,0 +1 @@ +Derive([("out","/nix/store/swkj0mrq0cq3dfli95v4am0427mi2hxf-advanced-attributes","","")],[("/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv",["dev","out"]),("/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv",["dev","out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__darwinAllowLocalNetworking","1"),("__impureHostDeps","/usr/bin/ditto"),("__noChroot","1"),("__sandboxProfile","sandcastle"),("allowSubstitutes",""),("allowedReferences","/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"),("allowedRequisites","/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"),("builder","/bin/bash"),("disallowedReferences","/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"),("disallowedRequisites","/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"),("impureEnvVars","UNICORN"),("name","advanced-attributes"),("out","/nix/store/swkj0mrq0cq3dfli95v4am0427mi2hxf-advanced-attributes"),("preferLocalBuild","1"),("requiredSystemFeatures","rainbow uid-range"),("system","my-system")]) \ No newline at end of file diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index d8c9f254d..b67a0964a 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -220,6 +220,13 @@ nix store gc nix registry list --flake-registry "file://$registry" --refresh | grepQuiet flake3 mv "$registry.tmp" "$registry" +# Ensure that locking ignores the user registry. +mkdir -p "$TEST_HOME/.config/nix" +ln -sfn "$registry" "$TEST_HOME/.config/nix/registry.json" +nix flake metadata flake1 +expectStderr 1 nix flake update --flake-registry '' --flake "$flake3Dir" | grepQuiet "cannot find flake 'flake:flake1' in the flake registries" +rm "$TEST_HOME/.config/nix/registry.json" + # Test whether flakes are registered as GC roots for offline use. # FIXME: use tarballs rather than git. rm -rf "$TEST_HOME/.cache" diff --git a/tests/functional/flakes/meson.build b/tests/functional/flakes/meson.build index b8c650db4..368c43876 100644 --- a/tests/functional/flakes/meson.build +++ b/tests/functional/flakes/meson.build @@ -28,6 +28,7 @@ suites += { 'commit-lock-file-summary.sh', 'non-flake-inputs.sh', 'relative-paths.sh', + 'relative-paths-lockfile.sh', 'symlink-paths.sh', 'debugger.sh', 'source-paths.sh', diff --git a/tests/functional/flakes/relative-paths-lockfile.sh b/tests/functional/flakes/relative-paths-lockfile.sh new file mode 100644 index 000000000..d91aedd16 --- /dev/null +++ b/tests/functional/flakes/relative-paths-lockfile.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +source ./common.sh + +requireGit + +# Test a "vendored" subflake dependency. This is a relative path flake +# which doesn't reference the root flake and has its own lock file. +# +# This might occur in a monorepo for example. The root flake.lock is +# populated from the dependency's flake.lock. + +rootFlake="$TEST_ROOT/flake1" +subflake="$rootFlake/sub" +depFlakeA="$TEST_ROOT/depFlakeA" +depFlakeB="$TEST_ROOT/depFlakeB" + +rm -rf "$rootFlake" +mkdir -p "$rootFlake" "$subflake" "$depFlakeA" "$depFlakeB" + +cat > "$depFlakeA/flake.nix" < "$depFlakeB/flake.nix" < "$subflake/flake.nix" < "$rootFlake/flake.nix" <flake.nix + cat >example/flake.nix < repl_output 2>&1 & +repl_pid=$! +exec 3>repl_fifo # Open fifo for writing +echo "changingThing" >&3 +for i in $(seq 1 1000); do + if grep -q "beforeChange" repl_output; then + break + fi + cat repl_output + sleep 0.1 +done +if [[ "$i" -eq 100 ]]; then + echo "Timed out waiting for beforeChange" + exit 1 +fi + sed -i 's/beforeChange/afterChange/' flake/flake.nix -echo ":reload" -echo "changingThing" -) | nix repl ./flake) -echo "$replResult" | grepQuiet -s beforeChange -echo "$replResult" | grepQuiet -s afterChange + +# Send reload and second command +echo ":reload" >&3 +echo "changingThing" >&3 +echo "exit" >&3 +exec 3>&- # Close fifo +wait $repl_pid # Wait for process to finish +grep -q "afterChange" repl_output # Test recursive printing and formatting # Normal output should print attributes in lexicographical order non-recursively