wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-07-06 05:01:48 +02:00
Code Activity

Merge branch 'master' into overlayfs-store

Browse source
This commit is contained in:
John Ericson 2024-03-18 16:41:29 -04:00
commit 18945e3f44
131 changed files with 1958 additions and 4473 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/libutil/current-process.cc
View file

@ -38,6 +38,11 @@ unsigned int getMaxCPU()
auto cpuMax = readFile(cpuFile);
auto cpuMaxParts = tokenizeString<std::vector<std::string>>(cpuMax, " \n");
if (cpuMaxParts.size() != 2) {
return 0;
}
auto quota = cpuMaxParts[0];
auto period = cpuMaxParts[1];
if (quota != "max")

32
src/libutil/experimental-features.cc
View file

@ -1,4 +1,5 @@
#include "experimental-features.hh"
#include "fmt.hh"
#include "util.hh"
#include "nlohmann/json.hpp"
@ -10,6 +11,7 @@ struct ExperimentalFeatureDetails
ExperimentalFeature tag;
std::string_view name;
std::string_view description;
std::string_view trackingUrl;
};
/**
@ -35,6 +37,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
[__contentAddressed](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed)
for details.
)",
.trackingUrl = "https://github.com/NixOS/nix/milestone/35",
},
{
.tag = Xp::ImpureDerivations,
@ -65,6 +68,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
This is a more explicit alternative to using [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime).
)",
.trackingUrl = "https://github.com/NixOS/nix/milestone/42",
},
{
.tag = Xp::Flakes,
@ -73,6 +77,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
Enable flakes. See the manual entry for [`nix
flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details.
)",
.trackingUrl = "https://github.com/NixOS/nix/milestone/27",
},
{
.tag = Xp::FetchTree,
@ -86,6 +91,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
Enabling just this feature serves as a "release candidate", allowing users to try it out in isolation.
)",
.trackingUrl = "https://github.com/NixOS/nix/milestone/31",
},
{
.tag = Xp::NixCommand,
@ -94,6 +100,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
Enable the new `nix` subcommands. See the manual on
[`nix`](@docroot@/command-ref/new-cli/nix.md) for details.
)",
.trackingUrl = "https://github.com/NixOS/nix/milestone/28",
},
{
.tag = Xp::GitHashing,
@ -102,6 +109,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
Allow creating (content-addressed) store objects which are hashed via Git's hashing algorithm.
These store objects will not be understandable by older versions of Nix.
)",
.trackingUrl = "https://github.com/NixOS/nix/milestone/41",
},
{
.tag = Xp::RecursiveNix,
@ -143,6 +151,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
already in the build inputs or built by a previous recursive Nix
call.
)",
.trackingUrl = "https://github.com/NixOS/nix/milestone/47",
},
{
.tag = Xp::NoUrlLiterals,
@ -184,6 +193,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
containing parameters have to be quoted anyway, and unquoted URLs
may confuse external tooling.
)",
.trackingUrl = "https://github.com/NixOS/nix/milestone/44",
},
{
.tag = Xp::FetchClosure,
@ -191,6 +201,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
.description = R"(
Enable the use of the [`fetchClosure`](@docroot@/language/builtins.md#builtins-fetchClosure) built-in function in the Nix language.
)",
.trackingUrl = "https://github.com/NixOS/nix/milestone/40",
},
{
.tag = Xp::ReplFlake,
@ -200,6 +211,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands.
)",
.trackingUrl = "https://github.com/NixOS/nix/milestone/32",
},
{
.tag = Xp::AutoAllocateUids,
@ -208,6 +220,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
Allows Nix to automatically pick UIDs for builds, rather than creating
`nixbld*` user accounts. See the [`auto-allocate-uids`](@docroot@/command-ref/conf-file.md#conf-auto-allocate-uids) setting for details.
)",
.trackingUrl = "https://github.com/NixOS/nix/milestone/34",
},
{
.tag = Xp::Cgroups,
@ -216,6 +229,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
Allows Nix to execute builds inside cgroups. See
the [`use-cgroups`](@docroot@/command-ref/conf-file.md#conf-use-cgroups) setting for details.
)",
.trackingUrl = "https://github.com/NixOS/nix/milestone/36",
},
{
.tag = Xp::DaemonTrustOverride,
@ -226,6 +240,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
useful for various experiments with `nix-daemon --stdio`
networking.
)",
.trackingUrl = "https://github.com/NixOS/nix/milestone/38",
},
{
.tag = Xp::DynamicDerivations,
@ -239,6 +254,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
- dependencies in derivations on the outputs of
derivations that are themselves derivations outputs.
)",
.trackingUrl = "https://github.com/NixOS/nix/milestone/39",
},
{
.tag = Xp::ParseTomlTimestamps,
@ -246,6 +262,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
.description = R"(
Allow parsing of timestamps in builtins.fromTOML.
)",
.trackingUrl = "https://github.com/NixOS/nix/milestone/45",
},
{
.tag = Xp::ReadOnlyLocalStore,
@ -253,6 +270,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
.description = R"(
Allow the use of the `read-only` parameter in [local store](@docroot@/store/types/local-store.md) URIs.
)",
.trackingUrl = "https://github.com/NixOS/nix/milestone/46",
},
{
.tag = Xp::LocalOverlayStore,
@ -260,7 +278,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
.description = R"(
Allow the use of [local overlay store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-overlay-store).
)",
.trackingUrl = ""https://github.com/NixOS/nix/milestone/50",
.trackingUrl = "https://github.com/NixOS/nix/milestone/50",
},
{
.tag = Xp::ConfigurableImpureEnv,
@ -268,6 +286,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
.description = R"(
Allow the use of the [impure-env](@docroot@/command-ref/conf-file.md#conf-impure-env) setting.
)",
.trackingUrl = "https://github.com/NixOS/nix/milestone/37",
},
{
.tag = Xp::MountedSSHStore,
@ -275,6 +294,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
.description = R"(
Allow the use of the [`mounted SSH store`](@docroot@/command-ref/new-cli/nix3-help-stores.html#experimental-ssh-store-with-filesytem-mounted).
)",
.trackingUrl = "https://github.com/NixOS/nix/milestone/43",
},
{
.tag = Xp::VerifiedFetches,
@ -282,6 +302,7 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
.description = R"(
Enables verification of git commit signatures through the [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) built-in.
)",
.trackingUrl = "https://github.com/NixOS/nix/milestone/48",
},
}};
@ -320,9 +341,12 @@ std::string_view showExperimentalFeature(const ExperimentalFeature tag)
nlohmann::json documentExperimentalFeatures()
{
StringMap res;
for (auto & xpFeature : xpFeatureDetails)
res[std::string { xpFeature.name }] =
trim(stripIndentation(xpFeature.description));
for (auto & xpFeature : xpFeatureDetails) {
std::stringstream docOss;
docOss << stripIndentation(xpFeature.description);
docOss << fmt("\nRefer to [%1% tracking issue](%2%) for feature tracking.", xpFeature.name, xpFeature.trackingUrl);
res[std::string{xpFeature.name}] = trim(docOss.str());
}
return (nlohmann::json) res;
}

2
src/libutil/file-content-address.cc
View file

@ -123,7 +123,7 @@ Hash hashPath(
case FileIngestionMethod::Git:
return git::dumpHash(ht, accessor, path, filter).hash;
}
assert(false);
}
}

5
src/libutil/file-system.cc
View file

@ -617,6 +617,11 @@ void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete)
}
}
void copyFile(const Path & oldPath, const Path & newPath, bool andDelete)
{
return copy(fs::directory_entry(fs::path(oldPath)), fs::path(newPath), andDelete);
}
void renameFile(const Path & oldName, const Path & newName)
{
fs::rename(oldName, newName);

7
src/libutil/file-system.hh
View file

@ -186,6 +186,13 @@ void renameFile(const Path & src, const Path & dst);
*/
void moveFile(const Path & src, const Path & dst);
/**
* Recursively copy the content of `oldPath` to `newPath`. If `andDelete` is
* `true`, then also remove `oldPath` (making this equivalent to `moveFile`, but
* with the guaranty that the destination will be “fresh”, with no stale inode
* or file descriptor pointing to it).
*/
void copyFile(const Path & oldPath, const Path & newPath, bool andDelete);
/**
* Automatic cleanup of resources.

4
src/libutil/fmt.hh
View file

@ -8,7 +8,6 @@
namespace nix {
namespace {
/**
* A helper for writing `boost::format` expressions.
*
@ -35,14 +34,13 @@ inline void formatHelper(F & f, const T & x, const Args & ... args)
/**
* Set the correct exceptions for `fmt`.
*/
void setExceptions(boost::format & fmt)
inline void setExceptions(boost::format & fmt)
{
fmt.exceptions(
boost::io::all_error_bits ^
boost::io::too_many_args_bit ^
boost::io::too_few_args_bit);
}
}
/**
* A helper for writing a `boost::format` expression to a string.

75
src/libutil/git.cc
View file

@ -56,31 +56,63 @@ void parseBlob(
FileSystemObjectSink & sink,
const Path & sinkPath,
Source & source,
bool executable,
BlobMode blobMode,
const ExperimentalFeatureSettings & xpSettings)
{
xpSettings.require(Xp::GitHashing);
sink.createRegularFile(sinkPath, [&](auto & crf) {
if (executable)
crf.isExecutable();
unsigned long long size = std::stoi(getStringUntil(source, 0));
unsigned long long size = std::stoi(getStringUntil(source, 0));
auto doRegularFile = [&](bool executable) {
sink.createRegularFile(sinkPath, [&](auto & crf) {
if (executable)
crf.isExecutable();
crf.preallocateContents(size);
crf.preallocateContents(size);
unsigned long long left = size;
std::string buf;
buf.reserve(65536);
unsigned long long left = size;
std::string buf;
buf.reserve(65536);
while (left) {
while (left) {
checkInterrupt();
buf.resize(std::min((unsigned long long)buf.capacity(), left));
source(buf);
crf(buf);
left -= buf.size();
}
});
};
switch (blobMode) {
case BlobMode::Regular:
doRegularFile(false);
break;
case BlobMode::Executable:
doRegularFile(true);
break;
case BlobMode::Symlink:
{
std::string target;
target.resize(size, '0');
target.reserve(size);
for (size_t n = 0; n < target.size();) {
checkInterrupt();
buf.resize(std::min((unsigned long long)buf.capacity(), left));
source(buf);
crf(buf);
left -= buf.size();
n += source.read(
const_cast<char *>(target.c_str()) + n,
target.size() - n);
}
});
sink.createSymlink(sinkPath, target);
break;
}
default:
assert(false);
}
}
void parseTree(
@ -142,7 +174,7 @@ void parse(
FileSystemObjectSink & sink,
const Path & sinkPath,
Source & source,
bool executable,
BlobMode rootModeIfBlob,
std::function<SinkHook> hook,
const ExperimentalFeatureSettings & xpSettings)
{
@ -152,7 +184,7 @@ void parse(
switch (type) {
case ObjectType::Blob:
parseBlob(sink, sinkPath, source, executable, xpSettings);
parseBlob(sink, sinkPath, source, rootModeIfBlob, xpSettings);
break;
case ObjectType::Tree:
parseTree(sink, sinkPath, source, hook, xpSettings);
@ -177,7 +209,7 @@ std::optional<Mode> convertMode(SourceAccessor::Type type)
void restore(FileSystemObjectSink & sink, Source & source, std::function<RestoreHook> hook)
{
parse(sink, "", source, false, [&](Path name, TreeEntry entry) {
parse(sink, "", source, BlobMode::Regular, [&](Path name, TreeEntry entry) {
auto [accessor, from] = hook(entry.hash);
auto stat = accessor->lstat(from);
auto gotOpt = convertMode(stat.type);
@ -275,6 +307,13 @@ Mode dump(
}
case SourceAccessor::tSymlink:
{
auto target = accessor.readLink(path);
dumpBlobPrefix(target.size(), sink, xpSettings);
sink(target);
return Mode::Symlink;
}
case SourceAccessor::tMisc:
default:
throw Error("file '%1%' has an unsupported type", path);

21
src/libutil/git.hh
View file

@ -75,10 +75,23 @@ ObjectType parseObjectType(
Source & source,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
/**
* These 3 modes are represented by blob objects.
*
* Sometimes we need this information to disambiguate how a blob is
* being used to better match our own "file system object" data model.
*/
enum struct BlobMode : RawMode
{
Regular = static_cast<RawMode>(Mode::Regular),
Executable = static_cast<RawMode>(Mode::Executable),
Symlink = static_cast<RawMode>(Mode::Symlink),
};
void parseBlob(
FileSystemObjectSink & sink, const Path & sinkPath,
Source & source,
bool executable,
BlobMode blobMode,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
void parseTree(
@ -89,11 +102,15 @@ void parseTree(
/**
* Helper putting the previous three `parse*` functions together.
*
* @rootModeIfBlob How to interpret a root blob, for which there is no
* disambiguating dir entry to answer that questino. If the root it not
* a blob, this is ignored.
*/
void parse(
FileSystemObjectSink & sink, const Path & sinkPath,
Source & source,
bool executable,
BlobMode rootModeIfBlob,
std::function<SinkHook> hook,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);

55
src/libutil/position.cc
View file

@ -29,32 +29,17 @@ std::optional<LinesOfCode> Pos::getCodeLines() const
return std::nullopt;
if (auto source = getSource()) {
std::istringstream iss(*source);
// count the newlines.
int count = 0;
std::string curLine;
int pl = line - 1;
LinesIterator lines(*source), end;
LinesOfCode loc;
do {
std::getline(iss, curLine);
++count;
if (count < pl)
;
else if (count == pl) {
loc.prevLineOfCode = curLine;
} else if (count == pl + 1) {
loc.errLineOfCode = curLine;
} else if (count == pl + 2) {
loc.nextLineOfCode = curLine;
break;
}
if (!iss.good())
break;
} while (true);
if (line > 1)
std::advance(lines, line - 2);
if (lines != end && line > 1)
loc.prevLineOfCode = *lines++;
if (lines != end)
loc.errLineOfCode = *lines++;
if (lines != end)
loc.nextLineOfCode = *lines++;
return loc;
}
@ -109,4 +94,26 @@ std::ostream & operator<<(std::ostream & str, const Pos & pos)
return str;
}
void Pos::LinesIterator::bump(bool atFirst)
{
if (!atFirst) {
pastEnd = input.empty();
if (!input.empty() && input[0] == '\r')
input.remove_prefix(1);
if (!input.empty() && input[0] == '\n')
input.remove_prefix(1);
}
// nix line endings are not only \n as eg std::getline assumes, but also
// \r\n **and \r alone**. not treating them all the same causes error
// reports to not match with line numbers as the parser expects them.
auto eol = input.find_first_of("\r\n");
if (eol > input.size())
eol = input.size();
curLine = input.substr(0, eol);
input.remove_prefix(eol);
}
}

42
src/libutil/position.hh
View file

@ -67,6 +67,48 @@ struct Pos
bool operator==(const Pos & rhs) const = default;
bool operator!=(const Pos & rhs) const = default;
bool operator<(const Pos & rhs) const;
struct LinesIterator {
using difference_type = size_t;
using value_type = std::string_view;
using reference = const std::string_view &;
using pointer = const std::string_view *;
using iterator_category = std::input_iterator_tag;
LinesIterator(): pastEnd(true) {}
explicit LinesIterator(std::string_view input): input(input), pastEnd(input.empty()) {
if (!pastEnd)
bump(true);
}
LinesIterator & operator++() {
bump(false);
return *this;
}
LinesIterator operator++(int) {
auto result = *this;
++*this;
return result;
}
reference operator*() const { return curLine; }
pointer operator->() const { return &curLine; }
bool operator!=(const LinesIterator & other) const {
return !(*this == other);
}
bool operator==(const LinesIterator & other) const {
return (pastEnd && other.pastEnd)
|| (std::forward_as_tuple(input.size(), input.data())
== std::forward_as_tuple(other.input.size(), other.input.data()));
}
private:
std::string_view input, curLine;
bool pastEnd = false;
void bump(bool atFirst);
};
};
std::ostream & operator<<(std::ostream & str, const Pos & pos);

12
src/libutil/posix-source-accessor.cc
View file

@ -85,16 +85,20 @@ bool PosixSourceAccessor::pathExists(const CanonPath & path)
std::optional<struct stat> PosixSourceAccessor::cachedLstat(const CanonPath & path)
{
static Sync<std::unordered_map<CanonPath, std::optional<struct stat>>> _cache;
static Sync<std::unordered_map<Path, std::optional<struct stat>>> _cache;
// Note: we convert std::filesystem::path to Path because the
// former is not hashable on libc++.
Path absPath = makeAbsPath(path);
{
auto cache(_cache.lock());
auto i = cache->find(path);
auto i = cache->find(absPath);
if (i != cache->end()) return i->second;
}
std::optional<struct stat> st{std::in_place};
if (::lstat(makeAbsPath(path).c_str(), &*st)) {
if (::lstat(absPath.c_str(), &*st)) {
if (errno == ENOENT || errno == ENOTDIR)
st.reset();
else
@ -103,7 +107,7 @@ std::optional<struct stat> PosixSourceAccessor::cachedLstat(const CanonPath & pa
auto cache(_cache.lock());
if (cache->size() >= 16384) cache->clear();
cache->emplace(path, st);
cache->emplace(absPath, st);
return st;
}

4
src/libutil/util.cc
View file

@ -9,6 +9,10 @@
#include <sodium.h>
#ifdef NDEBUG
#error "Nix may not be built with assertions disabled (i.e. with -DNDEBUG)."
#endif
namespace nix {
void initLibUtil() {