wroclaw-git-backups/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-07-07 10:11:47 +02:00
Code Activity

Merge remote-tracking branch 'upstream/master' into overlayfs-store

Browse source
This commit is contained in:
John Ericson 2024-02-01 11:07:47 -05:00
commit 31881d651a
378 changed files with 9552 additions and 4741 deletions
Download patch file Download diff file Expand all files Collapse all files

152
src/libutil/archive.cc
View file

@ -133,21 +133,21 @@ static SerialisationError badArchive(const std::string & s)
}
static void parseContents(ParseSink & sink, Source & source, const Path & path)
static void parseContents(CreateRegularFileSink & sink, Source & source)
{
uint64_t size = readLongLong(source);
sink.preallocateContents(size);
uint64_t left = size;
std::vector<char> buf(65536);
std::array<char, 65536> buf;
while (left) {
checkInterrupt();
auto n = buf.size();
if ((uint64_t)n > left) n = left;
source(buf.data(), n);
sink.receiveContents({buf.data(), n});
sink({buf.data(), n});
left -= n;
}
@ -164,109 +164,121 @@ struct CaseInsensitiveCompare
};
static void parse(ParseSink & sink, Source & source, const Path & path)
static void parse(FileSystemObjectSink & sink, Source & source, const Path & path)
{
std::string s;
s = readString(source);
if (s != "(") throw badArchive("expected open tag");
enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown;
std::map<Path, int, CaseInsensitiveCompare> names;
while (1) {
auto getString = [&]() {
checkInterrupt();
return readString(source);
};
s = readString(source);
// For first iteration
s = getString();
while (1) {
if (s == ")") {
break;
}
else if (s == "type") {
if (type != tpUnknown)
throw badArchive("multiple type fields");
std::string t = readString(source);
std::string t = getString();
if (t == "regular") {
type = tpRegular;
sink.createRegularFile(path);
sink.createRegularFile(path, [&](auto & crf) {
while (1) {
s = getString();
if (s == "contents") {
parseContents(crf, source);
}
else if (s == "executable") {
auto s2 = getString();
if (s2 != "") throw badArchive("executable marker has non-empty value");
crf.isExecutable();
}
else break;
}
});
}
else if (t == "directory") {
sink.createDirectory(path);
type = tpDirectory;
while (1) {
s = getString();
if (s == "entry") {
std::string name, prevName;
s = getString();
if (s != "(") throw badArchive("expected open tag");
while (1) {
s = getString();
if (s == ")") {
break;
} else if (s == "name") {
name = getString();
if (name.empty() || name == "." || name == ".." || name.find('/') != std::string::npos || name.find((char) 0) != std::string::npos)
throw Error("NAR contains invalid file name '%1%'", name);
if (name <= prevName)
throw Error("NAR directory is not sorted");
prevName = name;
if (archiveSettings.useCaseHack) {
auto i = names.find(name);
if (i != names.end()) {
debug("case collision between '%1%' and '%2%'", i->first, name);
name += caseHackSuffix;
name += std::to_string(++i->second);
} else
names[name] = 0;
}
} else if (s == "node") {
if (name.empty()) throw badArchive("entry name missing");
parse(sink, source, path + "/" + name);
} else
throw badArchive("unknown field " + s);
}
}
else break;
}
}
else if (t == "symlink") {
type = tpSymlink;
s = getString();
if (s != "target")
throw badArchive("expected 'target' got " + s);
std::string target = getString();
sink.createSymlink(path, target);
// for the next iteration
s = getString();
}
else throw badArchive("unknown file type " + t);
}
else if (s == "contents" && type == tpRegular) {
parseContents(sink, source, path);
sink.closeRegularFile();
}
else if (s == "executable" && type == tpRegular) {
auto s = readString(source);
if (s != "") throw badArchive("executable marker has non-empty value");
sink.isExecutable();
}
else if (s == "entry" && type == tpDirectory) {
std::string name, prevName;
s = readString(source);
if (s != "(") throw badArchive("expected open tag");
while (1) {
checkInterrupt();
s = readString(source);
if (s == ")") {
break;
} else if (s == "name") {
name = readString(source);
if (name.empty() || name == "." || name == ".." || name.find('/') != std::string::npos || name.find((char) 0) != std::string::npos)
throw Error("NAR contains invalid file name '%1%'", name);
if (name <= prevName)
throw Error("NAR directory is not sorted");
prevName = name;
if (archiveSettings.useCaseHack) {
auto i = names.find(name);
if (i != names.end()) {
debug("case collision between '%1%' and '%2%'", i->first, name);
name += caseHackSuffix;
name += std::to_string(++i->second);
} else
names[name] = 0;
}
} else if (s == "node") {
if (name.empty()) throw badArchive("entry name missing");
parse(sink, source, path + "/" + name);
} else
throw badArchive("unknown field " + s);
}
}
else if (s == "target" && type == tpSymlink) {
std::string target = readString(source);
sink.createSymlink(path, target);
}
else
throw badArchive("unknown field " + s);
}
}
void parseDump(ParseSink & sink, Source & source)
void parseDump(FileSystemObjectSink & sink, Source & source)
{
std::string version;
try {
@ -294,7 +306,7 @@ void copyNAR(Source & source, Sink & sink)
// FIXME: if 'source' is the output of dumpPath() followed by EOF,
// we should just forward all data directly without parsing.
NullParseSink parseSink; /* just parse the NAR */
NullFileSystemObjectSink parseSink; /* just parse the NAR */
TeeSource wrapper { source, sink };

2
src/libutil/archive.hh
View file

@ -73,7 +73,7 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink,
*/
void dumpString(std::string_view s, Sink & sink);
void parseDump(ParseSink & sink, Source & source);
void parseDump(FileSystemObjectSink & sink, Source & source);
void restorePath(const Path & path, Source & source);

10
src/libutil/args.cc
View file

@ -304,7 +304,7 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang)
for (auto pos = savedArgs.begin(); pos != savedArgs.end();pos++)
cmdline.push_back(*pos);
}
} catch (SysError &) { }
} catch (SystemError &) { }
}
for (auto pos = cmdline.begin(); pos != cmdline.end(); ) {
@ -557,7 +557,7 @@ Args::Flag Args::Flag::mkHashFormatFlagWithDefault(std::string &&longName, HashF
assert(*hf == nix::HashFormat::SRI);
return Flag{
.longName = std::move(longName),
.description = "hash format ('base16', 'nix32', 'base64', 'sri'). Default: 'sri'",
.description = "Hash format (`base16`, `nix32`, `base64`, `sri`). Default: `sri`.",
.labels = {"hash-format"},
.handler = {[hf](std::string s) {
*hf = parseHashFormat(s);
@ -569,7 +569,7 @@ Args::Flag Args::Flag::mkHashFormatFlagWithDefault(std::string &&longName, HashF
Args::Flag Args::Flag::mkHashFormatOptFlag(std::string && longName, std::optional<HashFormat> * ohf) {
return Flag{
.longName = std::move(longName),
.description = "hash format ('base16', 'nix32', 'base64', 'sri').",
.description = "Hash format (`base16`, `nix32`, `base64`, `sri`).",
.labels = {"hash-format"},
.handler = {[ohf](std::string s) {
*ohf = std::optional<HashFormat>{parseHashFormat(s)};
@ -589,7 +589,7 @@ Args::Flag Args::Flag::mkHashAlgoFlag(std::string && longName, HashAlgorithm * h
{
return Flag{
.longName = std::move(longName),
.description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')",
.description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`).",
.labels = {"hash-algo"},
.handler = {[ha](std::string s) {
*ha = parseHashAlgo(s);
@ -602,7 +602,7 @@ Args::Flag Args::Flag::mkHashAlgoOptFlag(std::string && longName, std::optional<
{
return Flag{
.longName = std::move(longName),
.description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.",
.description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`). Can be omitted for SRI hashes.",
.labels = {"hash-algo"},
.handler = {[oha](std::string s) {
*oha = std::optional<HashAlgorithm>{parseHashAlgo(s)};

6
src/libutil/args.hh
View file

@ -177,7 +177,13 @@ protected:
std::optional<ExperimentalFeature> experimentalFeature;
static Flag mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha);
static Flag mkHashAlgoFlag(HashAlgorithm * ha) {
return mkHashAlgoFlag("hash-algo", ha);
}
static Flag mkHashAlgoOptFlag(std::string && longName, std::optional<HashAlgorithm> * oha);
static Flag mkHashAlgoOptFlag(std::optional<HashAlgorithm> * oha) {
return mkHashAlgoOptFlag("hash-algo", oha);
}
static Flag mkHashFormatFlagWithDefault(std::string && longName, HashFormat * hf);
static Flag mkHashFormatOptFlag(std::string && longName, std::optional<HashFormat> * ohf);
};

6
src/libutil/canon-path.cc
View file

@ -6,11 +6,11 @@ namespace nix {
CanonPath CanonPath::root = CanonPath("/");
CanonPath::CanonPath(std::string_view raw)
: path(absPath((Path) raw, "/"))
: path(absPath(raw, "/"))
{ }
CanonPath::CanonPath(std::string_view raw, const CanonPath & root)
: path(absPath((Path) raw, root.abs()))
: path(absPath(raw, root.abs()))
{ }
CanonPath::CanonPath(const std::vector<std::string> & elems)
@ -22,7 +22,7 @@ CanonPath::CanonPath(const std::vector<std::string> & elems)
CanonPath CanonPath::fromCwd(std::string_view path)
{
return CanonPath(unchecked_t(), absPath((Path) path));
return CanonPath(unchecked_t(), absPath(path));
}
std::optional<CanonPath> CanonPath::parent() const

7
src/libutil/canon-path.hh
View file

@ -88,6 +88,13 @@ public:
std::string_view rel() const
{ return ((std::string_view) path).substr(1); }
const char * rel_c_str() const
{
auto cs = path.c_str();
assert(cs[0]); // for safety if invariant is broken
return &cs[1];
}
struct Iterator
{
std::string_view remaining;

2
src/libutil/cgroup.cc
View file

@ -95,7 +95,7 @@ static CgroupStats destroyCgroup(const Path & cgroup, bool returnStats)
using namespace std::string_literals;
warn("killing stray builder process %d (%s)...",
pid, trim(replaceStrings(cmdline, "\0"s, " ")));
} catch (SysError &) {
} catch (SystemError &) {
}
}
// FIXME: pid wraparound

4
src/libutil/comparator.hh
View file

@ -13,9 +13,9 @@
#define GENERATE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE, ...) \
PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const { \
__VA_OPT__(const MY_TYPE * me = this;) \
auto fields1 = std::make_tuple( __VA_ARGS__ ); \
auto fields1 = std::tie( __VA_ARGS__ ); \
__VA_OPT__(me = &other;) \
auto fields2 = std::make_tuple( __VA_ARGS__ ); \
auto fields2 = std::tie( __VA_ARGS__ ); \
return fields1 COMPARATOR fields2; \
}
#define GENERATE_EQUAL(prefix, qualification, my_type, args...) \

2
src/libutil/config.cc
View file

@ -124,7 +124,7 @@ static void applyConfigInner(const std::string & contents, const std::string & p
try {
std::string includedContents = readFile(path);
applyConfigInner(includedContents, p, parsedContents);
} catch (SysError &) {
} catch (SystemError &) {
// TODO: Do we actually want to ignore this? Or is it better to fail?
}
} else if (!ignoreMissing) {

26
src/libutil/current-process.cc
View file

@ -1,3 +1,6 @@
#include <algorithm>
#include <cstring>
#include "current-process.hh"
#include "namespaces.hh"
#include "util.hh"
@ -49,20 +52,27 @@ unsigned int getMaxCPU()
//////////////////////////////////////////////////////////////////////
#if __linux__
rlim_t savedStackSize = 0;
#endif
void setStackSize(size_t stackSize)
void setStackSize(rlim_t stackSize)
{
#if __linux__
struct rlimit limit;
if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) {
savedStackSize = limit.rlim_cur;
limit.rlim_cur = stackSize;
setrlimit(RLIMIT_STACK, &limit);
limit.rlim_cur = std::min(stackSize, limit.rlim_max);
if (setrlimit(RLIMIT_STACK, &limit) != 0) {
logger->log(
lvlError,
hintfmt(
"Failed to increase stack size from %1% to %2% (maximum allowed stack size: %3%): %4%",
savedStackSize,
stackSize,
limit.rlim_max,
std::strerror(errno)
).str()
);
}
}
#endif
}
void restoreProcessContext(bool restoreMounts)
@ -72,7 +82,6 @@ void restoreProcessContext(bool restoreMounts)
restoreMountNamespace();
}
#if __linux__
if (savedStackSize) {
struct rlimit limit;
if (getrlimit(RLIMIT_STACK, &limit) == 0) {
@ -80,7 +89,6 @@ void restoreProcessContext(bool restoreMounts)
setrlimit(RLIMIT_STACK, &limit);
}
}
#endif
}

3
src/libutil/current-process.hh
View file

@ -2,6 +2,7 @@
///@file
#include <optional>
#include <sys/resource.h>
#include "types.hh"
@ -16,7 +17,7 @@ unsigned int getMaxCPU();
/**
* Change the stack size.
*/
void setStackSize(size_t stackSize);
void setStackSize(rlim_t stackSize);
/**
* Restore the original inherited Unix process context (such as signal

18
src/libutil/english.cc Normal file
View file

@ -0,0 +1,18 @@
#include "english.hh"
namespace nix {
std::ostream & pluralize(
std::ostream & output,
unsigned int count,
const std::string_view single,
const std::string_view plural)
{
if (count == 1)
output << "1 " << single;
else
output << count << " " << plural;
return output;
}
}

18
src/libutil/english.hh Normal file
View file

@ -0,0 +1,18 @@
#pragma once
#include <iostream>
namespace nix {
/**
* Pluralize a given value.
*
* If `count == 1`, prints `1 {single}` to `output`, otherwise prints `{count} {plural}`.
*/
std::ostream & pluralize(
std::ostream & output,
unsigned int count,
const std::string_view single,
const std::string_view plural);
}

165
src/libutil/error.cc
View file

@ -2,6 +2,7 @@
#include "environment-variables.hh"
#include "signals.hh"
#include "terminal.hh"
#include "position.hh"
#include <iostream>
#include <optional>
@ -10,7 +11,7 @@
namespace nix {
void BaseError::addTrace(std::shared_ptr<AbstractPos> && e, hintformat hint, bool frame)
void BaseError::addTrace(std::shared_ptr<Pos> && e, hintformat hint, bool frame)
{
err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .frame = frame });
}
@ -41,58 +42,36 @@ std::ostream & operator <<(std::ostream & os, const hintformat & hf)
return os << hf.str();
}
std::ostream & operator <<(std::ostream & str, const AbstractPos & pos)
/**
* An arbitrarily defined value comparison for the purpose of using traces in the key of a sorted container.
*/
inline bool operator<(const Trace& lhs, const Trace& rhs)
{
pos.print(str);
str << ":" << pos.line;
if (pos.column > 0)
str << ":" << pos.column;
return str;
}
std::optional<LinesOfCode> AbstractPos::getCodeLines() const
{
if (line == 0)
return std::nullopt;
if (auto source = getSource()) {
std::istringstream iss(*source);
// count the newlines.
int count = 0;
std::string curLine;
int pl = line - 1;
LinesOfCode loc;
do {
std::getline(iss, curLine);
++count;
if (count < pl)
;
else if (count == pl) {
loc.prevLineOfCode = curLine;
} else if (count == pl + 1) {
loc.errLineOfCode = curLine;
} else if (count == pl + 2) {
loc.nextLineOfCode = curLine;
break;
}
if (!iss.good())
break;
} while (true);
return loc;
// `std::shared_ptr` does not have value semantics for its comparison
// functions, so we need to check for nulls and compare the dereferenced
// values here.
if (lhs.pos != rhs.pos) {
if (!lhs.pos)
return true;
if (!rhs.pos)
return false;
if (*lhs.pos != *rhs.pos)
return *lhs.pos < *rhs.pos;
}
return std::nullopt;
// This formats a freshly formatted hint string and then throws it away, which
// shouldn't be much of a problem because it only runs when pos is equal, and this function is
// used for trace printing, which is infrequent.
return std::forward_as_tuple(lhs.hint.str(), lhs.frame)
< std::forward_as_tuple(rhs.hint.str(), rhs.frame);
}
inline bool operator> (const Trace& lhs, const Trace& rhs) { return rhs < lhs; }
inline bool operator<=(const Trace& lhs, const Trace& rhs) { return !(lhs > rhs); }
inline bool operator>=(const Trace& lhs, const Trace& rhs) { return !(lhs < rhs); }
// print lines of code to the ostream, indicating the error column.
void printCodeLines(std::ostream & out,
const std::string & prefix,
const AbstractPos & errPos,
const Pos & errPos,
const LinesOfCode & loc)
{
// previous line of code.
@ -170,13 +149,12 @@ static bool printUnknownLocations = getEnv("_NIX_EVAL_SHOW_UNKNOWN_LOCATIONS").h
*
* @return true if a position was printed.
*/
static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std::shared_ptr<AbstractPos> & pos) {
static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std::shared_ptr<Pos> & pos) {
bool hasPos = pos && *pos;
if (hasPos) {
oss << "\n" << indent << ANSI_BLUE << "at " ANSI_WARNING << *pos << ANSI_NORMAL << ":";
oss << indent << ANSI_BLUE << "at " ANSI_WARNING << *pos << ANSI_NORMAL << ":";
if (auto loc = pos->getCodeLines()) {
oss << "\n";
printCodeLines(oss, "", *pos, *loc);
oss << "\n";
}
@ -186,6 +164,69 @@ static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std
return hasPos;
}
void printTrace(
std::ostream & output,
const std::string_view & indent,
size_t & count,
const Trace & trace)
{
output << "\n" << "… " << trace.hint.str() << "\n";
if (printPosMaybe(output, indent, trace.pos))
count++;
}
void printSkippedTracesMaybe(
std::ostream & output,
const std::string_view & indent,
size_t & count,
std::vector<Trace> & skippedTraces,
std::set<Trace> tracesSeen)
{
if (skippedTraces.size() > 0) {
// If we only skipped a few frames, print them out normally;
// messages like "1 duplicate frames omitted" aren't helpful.
if (skippedTraces.size() <= 5) {
for (auto & trace : skippedTraces) {
printTrace(output, indent, count, trace);
}
} else {
output << "\n" << ANSI_WARNING "(" << skippedTraces.size() << " duplicate frames omitted)" ANSI_NORMAL << "\n";
// Clear the set of "seen" traces after printing a chunk of
// `duplicate frames omitted`.
//
// Consider a mutually recursive stack trace with:
// - 10 entries of A
// - 10 entries of B
// - 10 entries of A
//
// If we don't clear `tracesSeen` here, we would print output like this:
// - 1 entry of A
// - (9 duplicate frames omitted)
// - 1 entry of B
// - (19 duplicate frames omitted)
//
// This would obscure the control flow, which went from A,
// to B, and back to A again.
//
// In contrast, if we do clear `tracesSeen`, the output looks like this:
// - 1 entry of A
// - (9 duplicate frames omitted)
// - 1 entry of B
// - (9 duplicate frames omitted)
// - 1 entry of A
// - (9 duplicate frames omitted)
//
// See: `tests/functional/lang/eval-fail-mutual-recursion.nix`
tracesSeen.clear();
}
}
// We've either printed each trace in `skippedTraces` normally, or
// printed a chunk of `duplicate frames omitted`. Either way, we've
// processed these traces and can clear them.
skippedTraces.clear();
}
std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace)
{
std::string prefix;
@ -294,7 +335,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
* try {
* e->eval(*this, env, v);
* if (v.type() != nAttrs)
* throwTypeError("value is %1% while a set was expected", v);
* throwTypeError("expected a set but found %1%", v);
* } catch (Error & e) {
* e.addTrace(pos, errorCtx);
* throw;
@ -308,7 +349,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
* e->eval(*this, env, v);
* try {
* if (v.type() != nAttrs)
* throwTypeError("value is %1% while a set was expected", v);
* throwTypeError("expected a set but found %1%", v);
* } catch (Error & e) {
* e.addTrace(pos, errorCtx);
* throw;
@ -334,7 +375,13 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
bool frameOnly = false;
if (!einfo.traces.empty()) {
// Stack traces seen since we last printed a chunk of `duplicate frames
// omitted`.
std::set<Trace> tracesSeen;
// A consecutive sequence of stack traces that are all in `tracesSeen`.
std::vector<Trace> skippedTraces;
size_t count = 0;
for (const auto & trace : einfo.traces) {
if (trace.hint.str().empty()) continue;
if (frameOnly && !trace.frame) continue;
@ -344,14 +391,21 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
break;
}
if (tracesSeen.count(trace)) {
skippedTraces.push_back(trace);
continue;
}
tracesSeen.insert(trace);
printSkippedTracesMaybe(oss, ellipsisIndent, count, skippedTraces, tracesSeen);
count++;
frameOnly = trace.frame;
oss << "\n" << "… " << trace.hint.str() << "\n";
if (printPosMaybe(oss, ellipsisIndent, trace.pos))
count++;
printTrace(oss, ellipsisIndent, count, trace);
}
printSkippedTracesMaybe(oss, ellipsisIndent, count, skippedTraces, tracesSeen);
oss << "\n" << prefix;
}
@ -370,4 +424,5 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
return out;
}
}

88
src/libutil/error.hh
View file

@ -25,6 +25,7 @@
#include <memory>
#include <map>
#include <optional>
#include <compare>
#include <sys/types.h>
#include <sys/stat.h>
@ -62,51 +63,28 @@ struct LinesOfCode {
std::optional<std::string> nextLineOfCode;
};
/**
* An abstract type that represents a location in a source file.
*/
struct AbstractPos
{
uint32_t line = 0;
uint32_t column = 0;
/**
* An AbstractPos may be a "null object", representing an unknown position.
*
* Return true if this position is known.
*/
inline operator bool() const { return line != 0; };
/**
* Return the contents of the source file.
*/
virtual std::optional<std::string> getSource() const
{ return std::nullopt; };
virtual void print(std::ostream & out) const = 0;
std::optional<LinesOfCode> getCodeLines() const;
virtual ~AbstractPos() = default;
};
std::ostream & operator << (std::ostream & str, const AbstractPos & pos);
struct Pos;
void printCodeLines(std::ostream & out,
const std::string & prefix,
const AbstractPos & errPos,
const Pos & errPos,
const LinesOfCode & loc);
struct Trace {
std::shared_ptr<AbstractPos> pos;
std::shared_ptr<Pos> pos;
hintformat hint;
bool frame;
};
inline bool operator<(const Trace& lhs, const Trace& rhs);
inline bool operator> (const Trace& lhs, const Trace& rhs);
inline bool operator<=(const Trace& lhs, const Trace& rhs);
inline bool operator>=(const Trace& lhs, const Trace& rhs);
struct ErrorInfo {
Verbosity level;
hintformat msg;
std::shared_ptr<AbstractPos> errPos;
std::shared_ptr<Pos> errPos;
std::list<Trace> traces;
Suggestions suggestions;
@ -177,12 +155,12 @@ public:
}
template<typename... Args>
void addTrace(std::shared_ptr<AbstractPos> && e, std::string_view fs, const Args & ... args)
void addTrace(std::shared_ptr<Pos> && e, std::string_view fs, const Args & ... args)
{
addTrace(std::move(e), hintfmt(std::string(fs), args...));
}
void addTrace(std::shared_ptr<AbstractPos> && e, hintformat hint, bool frame = false);
void addTrace(std::shared_ptr<Pos> && e, hintformat hint, bool frame = false);
bool hasTrace() const { return !err.traces.empty(); }
@ -200,20 +178,50 @@ MakeError(Error, BaseError);
MakeError(UsageError, Error);
MakeError(UnimplementedError, Error);
class SysError : public Error
/**
* To use in catch-blocks.
*/
MakeError(SystemError, Error);
/**
* POSIX system error, created using `errno`, `strerror` friends.
*
* Throw this, but prefer not to catch this, and catch `SystemError`
* instead. This allows implementations to freely switch between this
* and `WinError` without breaking catch blocks.
*
* However, it is permissible to catch this and rethrow so long as
* certain conditions are not met (e.g. to catch only if `errNo =
* EFooBar`). In that case, try to also catch the equivalent `WinError`
* code.
*
* @todo Rename this to `PosixError` or similar. At this point Windows
* support is too WIP to justify the code churn, but if it is finished
* then a better identifier becomes moe worth it.
*/
class SysError : public SystemError
{
public:
int errNo;
/**
* Construct using the explicitly-provided error number. `strerror`
* will be used to try to add additional information to the message.
*/
template<typename... Args>
SysError(int errNo_, const Args & ... args)
: Error("")
SysError(int errNo, const Args & ... args)
: SystemError(""), errNo(errNo)
{
errNo = errNo_;
auto hf = hintfmt(args...);
err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo));
}
/**
* Construct using the ambient `errno`.
*
* Be sure to not perform another `errno`-modifying operation before
* calling this constructor!
*/
template<typename... Args>
SysError(const Args & ... args)
: SysError(errno, args ...)
@ -221,7 +229,9 @@ public:
}
};
/** Throw an exception for the purpose of checking that exception handling works; see 'initLibUtil()'.
/**
* Throw an exception for the purpose of checking that exception
* handling works; see 'initLibUtil()'.
*/
void throwExceptionSelfCheck();

49
src/libutil/file-content-address.cc Normal file
View file

@ -0,0 +1,49 @@
#include "file-content-address.hh"
#include "archive.hh"
namespace nix {
void dumpPath(
SourceAccessor & accessor, const CanonPath & path,
Sink & sink,
FileIngestionMethod method,
PathFilter & filter)
{
switch (method) {
case FileIngestionMethod::Flat:
accessor.readFile(path, sink);
break;
case FileIngestionMethod::Recursive:
accessor.dumpPath(path, sink, filter);
break;
}
}
void restorePath(
const Path & path,
Source & source,
FileIngestionMethod method)
{
switch (method) {
case FileIngestionMethod::Flat:
writeFile(path, source);
break;
case FileIngestionMethod::Recursive:
restorePath(path, source);
break;
}
}
HashResult hashPath(
SourceAccessor & accessor, const CanonPath & path,
FileIngestionMethod method, HashAlgorithm ht,
PathFilter & filter)
{
HashSink sink { ht };
dumpPath(accessor, path, sink, method, filter);
return sink.finish();
}
}

56
src/libutil/file-content-address.hh Normal file
View file

@ -0,0 +1,56 @@
#pragma once
///@file
#include "source-accessor.hh"
#include "fs-sink.hh"
#include "util.hh"
namespace nix {
/**
* An enumeration of the main ways we can serialize file system
* objects.
*/
enum struct FileIngestionMethod : uint8_t {
/**
* Flat-file hashing. Directly ingest the contents of a single file
*/
Flat = 0,
/**
* Recursive (or NAR) hashing. Serializes the file-system object in
* Nix Archive format and ingest that.
*/
Recursive = 1,
};
/**
* Dump a serialization of the given file system object.
*/
void dumpPath(
SourceAccessor & accessor, const CanonPath & path,
Sink & sink,
FileIngestionMethod method,
PathFilter & filter = defaultPathFilter);
/**
* Restore a serialization of the given file system object.
*
* @TODO use an arbitrary `FileSystemObjectSink`.
*/
void restorePath(
const Path & path,
Source & source,
FileIngestionMethod method);
/**
* Compute the hash of the given file system object according to the
* given method.
*
* The hash is defined as (essentially) hashString(ht, dumpPath(path)).
*/
HashResult hashPath(
SourceAccessor & accessor, const CanonPath & path,
FileIngestionMethod method, HashAlgorithm ht,
PathFilter & filter = defaultPathFilter);
}

6
src/libutil/file-descriptor.cc
View file

@ -96,7 +96,7 @@ void drainFD(int fd, Sink & sink, bool block)
throw SysError("making file descriptor non-blocking");
}
Finally finally([&]() {
Finally finally([&] {
if (!block) {
if (fcntl(fd, F_SETFL, saved) == -1)
throw SysError("making file descriptor blocking");
@ -114,7 +114,7 @@ void drainFD(int fd, Sink & sink, bool block)
throw SysError("reading from file");
}
else if (rd == 0) break;
else sink({(char *) buf.data(), (size_t) rd});
else sink({reinterpret_cast<char *>(buf.data()), size_t(rd)});
}
}
@ -231,7 +231,7 @@ void closeMostFDs(const std::set<int> & exceptions)
}
}
return;
} catch (SysError &) {
} catch (SystemError &) {
}
#endif

28
src/libutil/file-system.cc
View file

@ -21,9 +21,16 @@ namespace fs = std::filesystem;
namespace nix {
Path absPath(Path path, std::optional<PathView> dir, bool resolveSymlinks)
Path absPath(PathView path, std::optional<PathView> dir, bool resolveSymlinks)
{
std::string scratch;
if (path[0] != '/') {
// In this case we need to call `canonPath` on a newly-created
// string. We set `scratch` to that string first, and then set
// `path` to `scratch`. This ensures the newly-created string
// lives long enough for the call to `canonPath`, and allows us
// to just accept a `std::string_view`.
if (!dir) {
#ifdef __GNU__
/* GNU (aka. GNU/Hurd) doesn't have any limitation on path
@ -35,12 +42,13 @@ Path absPath(Path path, std::optional<PathView> dir, bool resolveSymlinks)
if (!getcwd(buf, sizeof(buf)))
#endif
throw SysError("cannot get cwd");
path = concatStrings(buf, "/", path);
scratch = concatStrings(buf, "/", path);
#ifdef __GNU__
free(buf);
#endif
} else
path = concatStrings(*dir, "/", path);
scratch = concatStrings(*dir, "/", path);
path = scratch;
}
return canonPath(path, resolveSymlinks);
}
@ -82,7 +90,7 @@ Path canonPath(PathView path, bool resolveSymlinks)
/* Normal component; copy it. */
else {
s += '/';
if (const auto slash = path.find('/'); slash == std::string::npos) {
if (const auto slash = path.find('/'); slash == path.npos) {
s += path;
path = {};
} else {
@ -108,14 +116,18 @@ Path canonPath(PathView path, bool resolveSymlinks)
}
}
return s.empty() ? "/" : std::move(s);
if (s.empty()) {
s = "/";
}
return s;
}
Path dirOf(const PathView path)
{
Path::size_type pos = path.rfind('/');
if (pos == std::string::npos)
if (pos == path.npos)
return ".";
return pos == 0 ? "/" : Path(path, 0, pos);
}
@ -131,7 +143,7 @@ std::string_view baseNameOf(std::string_view path)
last -= 1;
auto pos = path.rfind('/', last);
if (pos == std::string::npos)
if (pos == path.npos)
pos = 0;
else
pos += 1;
@ -307,7 +319,7 @@ void writeFile(const Path & path, Source & source, mode_t mode, bool sync)
if (!fd)
throw SysError("opening file '%1%'", path);
std::vector<char> buf(64 * 1024);
std::array<char, 64 * 1024> buf;
try {
while (true) {

2
src/libutil/file-system.hh
View file

@ -41,7 +41,7 @@ struct Source;
* specified directory, or the current directory otherwise. The path
* is also canonicalised.
*/
Path absPath(Path path,
Path absPath(PathView path,
std::optional<PathView> dir = {},
bool resolveSymlinks = false);

74
src/libutil/fs-sink.cc
View file

@ -7,7 +7,7 @@ namespace nix {
void copyRecursive(
SourceAccessor & accessor, const CanonPath & from,
ParseSink & sink, const Path & to)
FileSystemObjectSink & sink, const Path & to)
{
auto stat = accessor.lstat(from);
@ -19,16 +19,12 @@ void copyRecursive(
case SourceAccessor::tRegular:
{
sink.createRegularFile(to);
if (stat.isExecutable)
sink.isExecutable();
LambdaSink sink2 {
[&](auto d) {
sink.receiveContents(d);
}
};
accessor.readFile(from, sink2, [&](uint64_t size) {
sink.preallocateContents(size);
sink.createRegularFile(to, [&](CreateRegularFileSink & crf) {
if (stat.isExecutable)
crf.isExecutable();
accessor.readFile(from, crf, [&](uint64_t size) {
crf.preallocateContents(size);
});
});
break;
}
@ -71,20 +67,24 @@ void RestoreSink::createDirectory(const Path & path)
throw SysError("creating directory '%1%'", p);
};
void RestoreSink::createRegularFile(const Path & path)
struct RestoreRegularFile : CreateRegularFileSink {
AutoCloseFD fd;
void operator () (std::string_view data) override;
void isExecutable() override;
void preallocateContents(uint64_t size) override;
};
void RestoreSink::createRegularFile(const Path & path, std::function<void(CreateRegularFileSink &)> func)
{
Path p = dstPath + path;
fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666);
if (!fd) throw SysError("creating file '%1%'", p);
RestoreRegularFile crf;
crf.fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666);
if (!crf.fd) throw SysError("creating file '%1%'", p);
func(crf);
}
void RestoreSink::closeRegularFile()
{
/* Call close explicitly to make sure the error is checked */
fd.close();
}
void RestoreSink::isExecutable()
void RestoreRegularFile::isExecutable()
{
struct stat st;
if (fstat(fd.get(), &st) == -1)
@ -93,7 +93,7 @@ void RestoreSink::isExecutable()
throw SysError("fchmod");
}
void RestoreSink::preallocateContents(uint64_t len)
void RestoreRegularFile::preallocateContents(uint64_t len)
{
if (!restoreSinkSettings.preallocateContents)
return;
@ -111,7 +111,7 @@ void RestoreSink::preallocateContents(uint64_t len)
#endif
}
void RestoreSink::receiveContents(std::string_view data)
void RestoreRegularFile::operator () (std::string_view data)
{
writeFull(fd.get(), data);
}
@ -122,4 +122,32 @@ void RestoreSink::createSymlink(const Path & path, const std::string & target)
nix::createSymlink(target, p);
}
void RegularFileSink::createRegularFile(const Path & path, std::function<void(CreateRegularFileSink &)> func)
{
struct CRF : CreateRegularFileSink {
RegularFileSink & back;
CRF(RegularFileSink & back) : back(back) {}
void operator () (std::string_view data) override
{
back.sink(data);
}
void isExecutable() override {}
} crf { *this };
func(crf);
}
void NullFileSystemObjectSink::createRegularFile(const Path & path, std::function<void(CreateRegularFileSink &)> func)
{
struct : CreateRegularFileSink {
void operator () (std::string_view data) override {}
void isExecutable() override {}
} crf;
// Even though `NullFileSystemObjectSink` doesn't do anything, it's important
// that we call the function, to e.g. advance the parser using this
// sink.
func(crf);
}
}

69
src/libutil/fs-sink.hh
View file

@ -9,18 +9,13 @@
namespace nix {
/**
* \todo Fix this API, it sucks.
* Actions on an open regular file in the process of creating it.
*
* See `FileSystemObjectSink::createRegularFile`.
*/
struct ParseSink
struct CreateRegularFileSink : Sink
{
virtual void createDirectory(const Path & path) = 0;
virtual void createRegularFile(const Path & path) = 0;
virtual void receiveContents(std::string_view data) = 0;
virtual void isExecutable() = 0;
virtual void closeRegularFile() = 0;
virtual void createSymlink(const Path & path, const std::string & target) = 0;
/**
* An optimization. By default, do nothing.
@ -28,46 +23,55 @@ struct ParseSink
virtual void preallocateContents(uint64_t size) { };
};
struct FileSystemObjectSink
{
virtual void createDirectory(const Path & path) = 0;
/**
* This function in general is no re-entrant. Only one file can be
* written at a time.
*/
virtual void createRegularFile(
const Path & path,
std::function<void(CreateRegularFileSink &)>) = 0;
virtual void createSymlink(const Path & path, const std::string & target) = 0;
};
/**
* Recusively copy file system objects from the source into the sink.
* Recursively copy file system objects from the source into the sink.
*/
void copyRecursive(
SourceAccessor & accessor, const CanonPath & sourcePath,
ParseSink & sink, const Path & destPath);
FileSystemObjectSink & sink, const Path & destPath);
/**
* Ignore everything and do nothing
*/
struct NullParseSink : ParseSink
struct NullFileSystemObjectSink : FileSystemObjectSink
{
void createDirectory(const Path & path) override { }
void receiveContents(std::string_view data) override { }
void createSymlink(const Path & path, const std::string & target) override { }
void createRegularFile(const Path & path) override { }
void closeRegularFile() override { }
void isExecutable() override { }
void createRegularFile(
const Path & path,
std::function<void(CreateRegularFileSink &)>) override;
};
/**
* Write files at the given path
*/
struct RestoreSink : ParseSink
struct RestoreSink : FileSystemObjectSink
{
Path dstPath;
void createDirectory(const Path & path) override;
void createRegularFile(const Path & path) override;
void receiveContents(std::string_view data) override;
void isExecutable() override;
void closeRegularFile() override;
void createRegularFile(
const Path & path,
std::function<void(CreateRegularFileSink &)>) override;
void createSymlink(const Path & path, const std::string & target) override;
void preallocateContents(uint64_t size) override;
private:
AutoCloseFD fd;
};
/**
@ -75,7 +79,7 @@ private:
* `receiveContents` to the underlying `Sink`. For anything but a single
* file, set `regular = true` so the caller can fail accordingly.
*/
struct RegularFileSink : ParseSink
struct RegularFileSink : FileSystemObjectSink
{
bool regular = true;
Sink & sink;
@ -87,19 +91,14 @@ struct RegularFileSink : ParseSink
regular = false;
}
void receiveContents(std::string_view data) override
{
sink(data);
}
void createSymlink(const Path & path, const std::string & target) override
{
regular = false;
}
void createRegularFile(const Path & path) override { }
void closeRegularFile() override { }
void isExecutable() override { }
void createRegularFile(
const Path & path,
std::function<void(CreateRegularFileSink &)>) override;
};
}

134
src/libutil/git.cc
View file

@ -52,24 +52,22 @@ static std::string getString(Source & source, int n)
return v;
}
void parse(
ParseSink & sink,
void parseBlob(
FileSystemObjectSink & sink,
const Path & sinkPath,
Source & source,
std::function<SinkHook> hook,
bool executable,
const ExperimentalFeatureSettings & xpSettings)
{
xpSettings.require(Xp::GitHashing);
auto type = getString(source, 5);
if (type == "blob ") {
sink.createRegularFile(sinkPath);
sink.createRegularFile(sinkPath, [&](auto & crf) {
if (executable)
crf.isExecutable();
unsigned long long size = std::stoi(getStringUntil(source, 0));
sink.preallocateContents(size);
crf.preallocateContents(size);
unsigned long long left = size;
std::string buf;
@ -79,47 +77,91 @@ void parse(
checkInterrupt();
buf.resize(std::min((unsigned long long)buf.capacity(), left));
source(buf);
sink.receiveContents(buf);
crf(buf);
left -= buf.size();
}
});
}
void parseTree(
FileSystemObjectSink & sink,
const Path & sinkPath,
Source & source,
std::function<SinkHook> hook,
const ExperimentalFeatureSettings & xpSettings)
{
unsigned long long size = std::stoi(getStringUntil(source, 0));
unsigned long long left = size;
sink.createDirectory(sinkPath);
while (left) {
std::string perms = getStringUntil(source, ' ');
left -= perms.size();
left -= 1;
RawMode rawMode = std::stoi(perms, 0, 8);
auto modeOpt = decodeMode(rawMode);
if (!modeOpt)
throw Error("Unknown Git permission: %o", perms);
auto mode = std::move(*modeOpt);
std::string name = getStringUntil(source, '\0');
left -= name.size();
left -= 1;
std::string hashs = getString(source, 20);
left -= 20;
Hash hash(HashAlgorithm::SHA1);
std::copy(hashs.begin(), hashs.end(), hash.hash);
hook(name, TreeEntry {
.mode = mode,
.hash = hash,
});
}
}
ObjectType parseObjectType(
Source & source,
const ExperimentalFeatureSettings & xpSettings)
{
xpSettings.require(Xp::GitHashing);
auto type = getString(source, 5);
if (type == "blob ") {
return ObjectType::Blob;
} else if (type == "tree ") {
unsigned long long size = std::stoi(getStringUntil(source, 0));
unsigned long long left = size;
sink.createDirectory(sinkPath);
while (left) {
std::string perms = getStringUntil(source, ' ');
left -= perms.size();
left -= 1;
RawMode rawMode = std::stoi(perms, 0, 8);
auto modeOpt = decodeMode(rawMode);
if (!modeOpt)
throw Error("Unknown Git permission: %o", perms);
auto mode = std::move(*modeOpt);
std::string name = getStringUntil(source, '\0');
left -= name.size();
left -= 1;
std::string hashs = getString(source, 20);
left -= 20;
Hash hash(HashAlgorithm::SHA1);
std::copy(hashs.begin(), hashs.end(), hash.hash);
hook(name, TreeEntry {
.mode = mode,
.hash = hash,
});
if (mode == Mode::Executable)
sink.isExecutable();
}
return ObjectType::Tree;
} else throw Error("input doesn't look like a Git object");
}
void parse(
FileSystemObjectSink & sink,
const Path & sinkPath,
Source & source,
bool executable,
std::function<SinkHook> hook,
const ExperimentalFeatureSettings & xpSettings)
{
xpSettings.require(Xp::GitHashing);
auto type = parseObjectType(source, xpSettings);
switch (type) {
case ObjectType::Blob:
parseBlob(sink, sinkPath, source, executable, xpSettings);
break;
case ObjectType::Tree:
parseTree(sink, sinkPath, source, hook, xpSettings);
break;
default:
assert(false);
};
}
std::optional<Mode> convertMode(SourceAccessor::Type type)
{
@ -133,9 +175,9 @@ std::optional<Mode> convertMode(SourceAccessor::Type type)
}
void restore(ParseSink & sink, Source & source, std::function<RestoreHook> hook)
void restore(FileSystemObjectSink & sink, Source & source, std::function<RestoreHook> hook)
{
parse(sink, "", source, [&](Path name, TreeEntry entry) {
parse(sink, "", source, false, [&](Path name, TreeEntry entry) {
auto [accessor, from] = hook(entry.hash);
auto stat = accessor->lstat(from);
auto gotOpt = convertMode(stat.type);

40
src/libutil/git.hh
View file

@ -13,12 +13,19 @@
namespace nix::git {
enum struct ObjectType {
Blob,
Tree,
//Commit,
//Tag,
};
using RawMode = uint32_t;
enum struct Mode : RawMode {
Directory = 0040000,
Executable = 0100755,
Regular = 0100644,
Executable = 0100755,
Symlink = 0120000,
};
@ -59,9 +66,34 @@ using Tree = std::map<std::string, TreeEntry>;
*/
using SinkHook = void(const Path & name, TreeEntry entry);
void parse(
ParseSink & sink, const Path & sinkPath,
/**
* Parse the "blob " or "tree " prefix.
*
* @throws if prefix not recognized
*/
ObjectType parseObjectType(
Source & source,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
void parseBlob(
FileSystemObjectSink & sink, const Path & sinkPath,
Source & source,
bool executable,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
void parseTree(
FileSystemObjectSink & sink, const Path & sinkPath,
Source & source,
std::function<SinkHook> hook,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
/**
* Helper putting the previous three `parse*` functions together.
*/
void parse(
FileSystemObjectSink & sink, const Path & sinkPath,
Source & source,
bool executable,
std::function<SinkHook> hook,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
@ -81,7 +113,7 @@ using RestoreHook = std::pair<SourceAccessor *, CanonPath>(Hash);
/**
* Wrapper around `parse` and `RestoreSink`
*/
void restore(ParseSink & sink, Source & source, std::function<RestoreHook> hook);
void restore(FileSystemObjectSink & sink, Source & source, std::function<RestoreHook> hook);
/**
* Dumps a single file to a sink

18
src/libutil/hash.cc
View file

@ -14,6 +14,8 @@
#include <sys/stat.h>
#include <fcntl.h>
#include <sodium.h>
namespace nix {
static size_t regularHashSize(HashAlgorithm type) {
@ -261,6 +263,13 @@ Hash::Hash(std::string_view rest, HashAlgorithm algo, bool isSRI)
throw BadHash("hash '%s' has wrong length for hash algorithm '%s'", rest, printHashAlgo(this->algo));
}
Hash Hash::random(HashAlgorithm algo)
{
Hash hash(algo);
randombytes_buf(hash.hash, hash.hashSize);
return hash;
}
Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashAlgorithm> ha)
{
if (hashStr.empty()) {
@ -367,15 +376,6 @@ HashResult HashSink::currentHash()
}
HashResult hashPath(
HashAlgorithm ha, const Path & path, PathFilter & filter)
{
HashSink sink(ha);
dumpPath(path, sink, filter);
return sink.finish();
}
Hash compressHash(const Hash & hash, unsigned int newSize)
{
Hash h(hash.algo);

13
src/libutil/hash.hh
View file

@ -5,7 +5,6 @@
#include "serialise.hh"
#include "file-system.hh"
namespace nix {
@ -143,6 +142,11 @@ public:
}
static Hash dummy;
/**
* @return a random hash with hash algorithm `algo`
*/
static Hash random(HashAlgorithm algo);
};
/**
@ -168,14 +172,11 @@ Hash hashString(HashAlgorithm ha, std::string_view s);
Hash hashFile(HashAlgorithm ha, const Path & path);
/**
* Compute the hash of the given path, serializing as a Nix Archive and
* then hashing that.
* The final hash and the number of bytes digested.
*
* The hash is defined as (essentially) hashString(ht, dumpPath(path)).
* @todo Convert to proper struct
*/
typedef std::pair<Hash, uint64_t> HashResult;
HashResult hashPath(HashAlgorithm ha, const Path & path,
PathFilter & filter = defaultPathFilter);
/**
* Compress a hash to the specified number of bytes by cyclically

27
src/libutil/input-accessor.hh Normal file
View file

@ -0,0 +1,27 @@
#pragma once
///@file
#include "source-accessor.hh"
#include "ref.hh"
#include "repair-flag.hh"
namespace nix {
MakeError(RestrictedPathError, Error);
struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<InputAccessor>
{
std::optional<std::string> fingerprint;
/**
* Return the maximum last-modified time of the files in this
* tree, if available.
*/
virtual std::optional<time_t> getLastModified()
{
return std::nullopt;
}
};
}

9
src/libutil/local.mk
View file

@ -4,15 +4,18 @@ libutil_NAME = libnixutil
libutil_DIR := $(d)
libutil_SOURCES := $(wildcard $(d)/*.cc)
libutil_SOURCES := $(wildcard $(d)/*.cc $(d)/signature/*.cc)
libutil_CXXFLAGS += -I src/libutil
libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
libutil_LDFLAGS += $(THREAD_LDFLAGS) $(LIBCURL_LIBS) $(SODIUM_LIBS) $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
$(foreach i, $(wildcard $(d)/args/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/args, 0644)))
$(foreach i, $(wildcard $(d)/signature/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/signature, 0644)))
ifeq ($(HAVE_LIBCPUID), 1)
libutil_LDFLAGS += -lcpuid
libutil_LDFLAGS += -lcpuid
endif

8
src/libutil/logging.cc
View file

@ -4,6 +4,8 @@
#include "terminal.hh"
#include "util.hh"
#include "config.hh"
#include "source-path.hh"
#include "position.hh"
#include <atomic>
#include <nlohmann/json.hpp>
@ -114,7 +116,7 @@ void writeToStderr(std::string_view s)
{
try {
writeFull(STDERR_FILENO, s, false);
} catch (SysError & e) {
} catch (SystemError & e) {
/* Ignore failing writes to stderr. We need to ignore write
errors to ensure that cleanup code that logs to stderr runs
to completion if the other side of stderr has been closed
@ -136,13 +138,13 @@ Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type,
logger.startActivity(id, lvl, type, s, fields, parent);
}
void to_json(nlohmann::json & json, std::shared_ptr<AbstractPos> pos)
void to_json(nlohmann::json & json, std::shared_ptr<Pos> pos)
{
if (pos) {
json["line"] = pos->line;
json["column"] = pos->column;
std::ostringstream str;
pos->print(str);
pos->print(str, true);
json["file"] = str.str();
} else {
json["line"] = nullptr;

39
src/libutil/memory-source-accessor.cc
View file

@ -134,36 +134,43 @@ void MemorySink::createDirectory(const Path & path)
throw Error("file '%s' is not a directory", path);
};
void MemorySink::createRegularFile(const Path & path)
struct CreateMemoryRegularFile : CreateRegularFileSink {
File::Regular & regularFile;
CreateMemoryRegularFile(File::Regular & r)
: regularFile(r)
{ }
void operator () (std::string_view data) override;
void isExecutable() override;
void preallocateContents(uint64_t size) override;
};
void MemorySink::createRegularFile(const Path & path, std::function<void(CreateRegularFileSink &)> func)
{
auto * f = dst.open(CanonPath{path}, File { File::Regular {} });
if (!f)
throw Error("file '%s' cannot be made because some parent file is not a directory", path);
if (!(r = std::get_if<File::Regular>(&f->raw)))
if (auto * rp = std::get_if<File::Regular>(&f->raw)) {
CreateMemoryRegularFile crf { *rp };
func(crf);
} else
throw Error("file '%s' is not a regular file", path);
}
void MemorySink::closeRegularFile()
void CreateMemoryRegularFile::isExecutable()
{
r = nullptr;
regularFile.executable = true;
}
void MemorySink::isExecutable()
void CreateMemoryRegularFile::preallocateContents(uint64_t len)
{
assert(r);
r->executable = true;
regularFile.contents.reserve(len);
}
void MemorySink::preallocateContents(uint64_t len)
void CreateMemoryRegularFile::operator () (std::string_view data)
{
assert(r);
r->contents.reserve(len);
}
void MemorySink::receiveContents(std::string_view data)
{
assert(r);
r->contents += data;
regularFile.contents += data;
}
void MemorySink::createSymlink(const Path & path, const std::string & target)

14
src/libutil/memory-source-accessor.hh
View file

@ -75,7 +75,7 @@ struct MemorySourceAccessor : virtual SourceAccessor
/**
* Write to a `MemorySourceAccessor` at the given path
*/
struct MemorySink : ParseSink
struct MemorySink : FileSystemObjectSink
{
MemorySourceAccessor & dst;
@ -83,17 +83,11 @@ struct MemorySink : ParseSink
void createDirectory(const Path & path) override;
void createRegularFile(const Path & path) override;
void receiveContents(std::string_view data) override;
void isExecutable() override;
void closeRegularFile() override;
void createRegularFile(
const Path & path,
std::function<void(CreateRegularFileSink &)>) override;
void createSymlink(const Path & path, const std::string & target) override;
void preallocateContents(uint64_t size) override;
private:
MemorySourceAccessor::File::Regular * r;
};
}

112
src/libutil/position.cc Normal file
View file

@ -0,0 +1,112 @@
#include "position.hh"
namespace nix {
Pos::Pos(const Pos * other)
{
if (!other) {
return;
}
line = other->line;
column = other->column;
origin = std::move(other->origin);
}
Pos::operator std::shared_ptr<Pos>() const
{
return std::make_shared<Pos>(&*this);
}
bool Pos::operator<(const Pos &rhs) const
{
return std::forward_as_tuple(line, column, origin)
< std::forward_as_tuple(rhs.line, rhs.column, rhs.origin);
}
std::optional<LinesOfCode> Pos::getCodeLines() const
{
if (line == 0)
return std::nullopt;
if (auto source = getSource()) {
std::istringstream iss(*source);
// count the newlines.
int count = 0;
std::string curLine;
int pl = line - 1;
LinesOfCode loc;
do {
std::getline(iss, curLine);
++count;
if (count < pl)
;
else if (count == pl) {
loc.prevLineOfCode = curLine;
} else if (count == pl + 1) {
loc.errLineOfCode = curLine;
} else if (count == pl + 2) {
loc.nextLineOfCode = curLine;
break;
}
if (!iss.good())
break;
} while (true);
return loc;
}
return std::nullopt;
}
std::optional<std::string> Pos::getSource() const
{
return std::visit(overloaded {
[](const std::monostate &) -> std::optional<std::string> {
return std::nullopt;
},
[](const Pos::Stdin & s) -> std::optional<std::string> {
// Get rid of the null terminators added by the parser.
return std::string(s.source->c_str());
},
[](const Pos::String & s) -> std::optional<std::string> {
// Get rid of the null terminators added by the parser.
return std::string(s.source->c_str());
},
[](const SourcePath & path) -> std::optional<std::string> {
try {
return path.readFile();
} catch (Error &) {
return std::nullopt;
}
}
}, origin);
}
void Pos::print(std::ostream & out, bool showOrigin) const
{
if (showOrigin) {
std::visit(overloaded {
[&](const std::monostate &) { out << "«none»"; },
[&](const Pos::Stdin &) { out << "«stdin»"; },
[&](const Pos::String & s) { out << "«string»"; },
[&](const SourcePath & path) { out << path; }
}, origin);
out << ":";
}
out << line;
if (column > 0)
out << ":" << column;
}
std::ostream & operator<<(std::ostream & str, const Pos & pos)
{
pos.print(str, true);
return str;
}
}

74
src/libutil/position.hh Normal file
View file

@ -0,0 +1,74 @@
#pragma once
/**
* @file
*
* @brief Pos and AbstractPos
*/
#include <cstdint>
#include <string>
#include "source-path.hh"
namespace nix {
/**
* A position and an origin for that position (like a source file).
*/
struct Pos
{
uint32_t line = 0;
uint32_t column = 0;
struct Stdin {
ref<std::string> source;
bool operator==(const Stdin & rhs) const
{ return *source == *rhs.source; }
bool operator!=(const Stdin & rhs) const
{ return *source != *rhs.source; }
bool operator<(const Stdin & rhs) const
{ return *source < *rhs.source; }
};
struct String {
ref<std::string> source;
bool operator==(const String & rhs) const
{ return *source == *rhs.source; }
bool operator!=(const String & rhs) const
{ return *source != *rhs.source; }
bool operator<(const String & rhs) const
{ return *source < *rhs.source; }
};
typedef std::variant<std::monostate, Stdin, String, SourcePath> Origin;
Origin origin = std::monostate();
Pos() { }
Pos(uint32_t line, uint32_t column, Origin origin)
: line(line), column(column), origin(origin) { }
Pos(Pos & other) = default;
Pos(const Pos & other) = default;
Pos(Pos && other) = default;
Pos(const Pos * other);
explicit operator bool() const { return line > 0; }
operator std::shared_ptr<Pos>() const;
/**
* Return the contents of the source file.
*/
std::optional<std::string> getSource() const;
void print(std::ostream & out, bool showOrigin) const;
std::optional<LinesOfCode> getCodeLines() const;
bool operator==(const Pos & rhs) const = default;
bool operator!=(const Pos & rhs) const = default;
bool operator<(const Pos & rhs) const;
};
std::ostream & operator<<(std::ostream & str, const Pos & pos);
}

2
src/libutil/posix-source-accessor.cc
View file

@ -25,7 +25,7 @@ void PosixSourceAccessor::readFile(
off_t left = st.st_size;
std::vector<unsigned char> buf(64 * 1024);
std::array<unsigned char, 64 * 1024> buf;
while (left) {
checkInterrupt();
ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size()));

25
src/libutil/processes.cc
View file

@ -131,7 +131,7 @@ void killUser(uid_t uid)
users to which the current process can send signals. So we
fork a process, switch to uid, and send a mass kill. */
Pid pid = startProcess([&]() {
Pid pid = startProcess([&] {
if (setuid(uid) == -1)
throw SysError("setting uid");
@ -168,11 +168,12 @@ void killUser(uid_t uid)
//////////////////////////////////////////////////////////////////////
using ChildWrapperFunction = std::function<void()>;
/* Wrapper around vfork to prevent the child process from clobbering
the caller's stack frame in the parent. */
static pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline));
static pid_t doFork(bool allowVfork, std::function<void()> fun)
static pid_t doFork(bool allowVfork, ChildWrapperFunction & fun) __attribute__((noinline));
static pid_t doFork(bool allowVfork, ChildWrapperFunction & fun)
{
#ifdef __linux__
pid_t pid = allowVfork ? vfork() : fork();
@ -188,8 +189,8 @@ static pid_t doFork(bool allowVfork, std::function<void()> fun)
#if __linux__
static int childEntry(void * arg)
{
auto main = (std::function<void()> *) arg;
(*main)();
auto & fun = *reinterpret_cast<ChildWrapperFunction*>(arg);
fun();
return 1;
}
#endif
@ -197,7 +198,7 @@ static int childEntry(void * arg)
pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
{
std::function<void()> wrapper = [&]() {
ChildWrapperFunction wrapper = [&] {
if (!options.allowVfork)
logger = makeSimpleLogger();
try {
@ -225,11 +226,11 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options)
assert(!(options.cloneFlags & CLONE_VM));
size_t stackSize = 1 * 1024 * 1024;
auto stack = (char *) mmap(0, stackSize,
PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
auto stack = static_cast<char *>(mmap(0, stackSize,
PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0));
if (stack == MAP_FAILED) throw SysError("allocating stack");
Finally freeStack([&]() { munmap(stack, stackSize); });
Finally freeStack([&] { munmap(stack, stackSize); });
pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper);
#else
@ -308,7 +309,7 @@ void runProgram2(const RunOptions & options)
}
/* Fork. */
Pid pid = startProcess([&]() {
Pid pid = startProcess([&] {
if (options.environment)
replaceEnv(*options.environment);
if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1)
@ -350,7 +351,7 @@ void runProgram2(const RunOptions & options)
std::promise<void> promise;
Finally doJoin([&]() {
Finally doJoin([&] {
if (writerThread.joinable())
writerThread.join();
});
@ -358,7 +359,7 @@ void runProgram2(const RunOptions & options)
if (source) {
in.readSide.close();
writerThread = std::thread([&]() {
writerThread = std::thread([&] {
try {
std::vector<char> buf(8 * 1024);
while (true) {

1
src/libutil/ref.hh
View file

@ -1,6 +1,7 @@
#pragma once
///@file
#include <compare>
#include <memory>
#include <exception>
#include <stdexcept>

8
src/libutil/repair-flag.hh Normal file
View file

@ -0,0 +1,8 @@
#pragma once
///@file
namespace nix {
enum RepairFlag : bool { NoRepair = false, Repair = true };
}

6
src/libutil/serialise.cc
View file

@ -53,7 +53,7 @@ void FdSink::writeUnbuffered(std::string_view data)
written += data.size();
try {
writeFull(fd, data);
} catch (SysError & e) {
} catch (SystemError & e) {
_good = false;
throw;
}
@ -82,7 +82,7 @@ void Source::operator () (std::string_view data)
void Source::drainInto(Sink & sink)
{
std::string s;
std::vector<char> buf(8192);
std::array<char, 8192> buf;
while (true) {
size_t n;
try {
@ -132,7 +132,7 @@ size_t FdSource::readUnbuffered(char * data, size_t len)
n = ::read(fd, data, len);
} while (n == -1 && errno == EINTR);
if (n == -1) { _good = false; throw SysError("reading from file"); }
if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); }
if (n == 0) { _good = false; throw EndOfFile(std::string(*endOfFileError)); }
read += n;
return n;
}

5
src/libutil/serialise.hh
View file

@ -153,12 +153,13 @@ struct FdSource : BufferedSource
{
int fd;
size_t read = 0;
BackedStringView endOfFileError{"unexpected end-of-file"};
FdSource() : fd(-1) { }
FdSource(int fd) : fd(fd) { }
FdSource(FdSource&&) = default;
FdSource(FdSource &&) = default;
FdSource& operator=(FdSource && s)
FdSource & operator=(FdSource && s)
{
fd = s.fd;
s.fd = -1;

2
src/libutil/signals.cc
View file

@ -179,7 +179,7 @@ std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()>
auto token = interruptCallbacks->nextToken++;
interruptCallbacks->callbacks.emplace(token, callback);
auto res = std::make_unique<InterruptCallbackImpl>();
std::unique_ptr<InterruptCallbackImpl> res {new InterruptCallbackImpl{}};
res->token = token;
return std::unique_ptr<InterruptCallback>(res.release());

105
src/libutil/signature/local-keys.cc Normal file
View file

@ -0,0 +1,105 @@
#include "signature/local-keys.hh"
#include "file-system.hh"
#include "util.hh"
#include <sodium.h>
namespace nix {
BorrowedCryptoValue BorrowedCryptoValue::parse(std::string_view s)
{
size_t colon = s.find(':');
if (colon == std::string::npos || colon == 0)
return {"", ""};
return {s.substr(0, colon), s.substr(colon + 1)};
}
Key::Key(std::string_view s)
{
auto ss = BorrowedCryptoValue::parse(s);
name = ss.name;
key = ss.payload;
if (name == "" || key == "")
throw Error("secret key is corrupt");
key = base64Decode(key);
}
std::string Key::to_string() const
{
return name + ":" + base64Encode(key);
}
SecretKey::SecretKey(std::string_view s)
: Key(s)
{
if (key.size() != crypto_sign_SECRETKEYBYTES)
throw Error("secret key is not valid");
}
std::string SecretKey::signDetached(std::string_view data) const
{
unsigned char sig[crypto_sign_BYTES];
unsigned long long sigLen;
crypto_sign_detached(sig, &sigLen, (unsigned char *) data.data(), data.size(),
(unsigned char *) key.data());
return name + ":" + base64Encode(std::string((char *) sig, sigLen));
}
PublicKey SecretKey::toPublicKey() const
{
unsigned char pk[crypto_sign_PUBLICKEYBYTES];
crypto_sign_ed25519_sk_to_pk(pk, (unsigned char *) key.data());
return PublicKey(name, std::string((char *) pk, crypto_sign_PUBLICKEYBYTES));
}
SecretKey SecretKey::generate(std::string_view name)
{
unsigned char pk[crypto_sign_PUBLICKEYBYTES];
unsigned char sk[crypto_sign_SECRETKEYBYTES];
if (crypto_sign_keypair(pk, sk) != 0)
throw Error("key generation failed");
return SecretKey(name, std::string((char *) sk, crypto_sign_SECRETKEYBYTES));
}
PublicKey::PublicKey(std::string_view s)
: Key(s)
{
if (key.size() != crypto_sign_PUBLICKEYBYTES)
throw Error("public key is not valid");
}
bool PublicKey::verifyDetached(std::string_view data, std::string_view sig) const
{
auto ss = BorrowedCryptoValue::parse(sig);
if (ss.name != std::string_view { name }) return false;
return verifyDetachedAnon(data, ss.payload);
}
bool PublicKey::verifyDetachedAnon(std::string_view data, std::string_view sig) const
{
auto sig2 = base64Decode(sig);
if (sig2.size() != crypto_sign_BYTES)
throw Error("signature is not valid");
return crypto_sign_verify_detached((unsigned char *) sig2.data(),
(unsigned char *) data.data(), data.size(),
(unsigned char *) key.data()) == 0;
}
bool verifyDetached(std::string_view data, std::string_view sig, const PublicKeys & publicKeys)
{
auto ss = BorrowedCryptoValue::parse(sig);
auto key = publicKeys.find(std::string(ss.name));
if (key == publicKeys.end()) return false;
return key->second.verifyDetachedAnon(data, ss.payload);
}
}

103
src/libutil/signature/local-keys.hh Normal file
View file

@ -0,0 +1,103 @@
#pragma once
///@file
#include "types.hh"
#include <map>
namespace nix {
/**
* Except where otherwise noted, Nix serializes keys and signatures in
* the form:
*
* ```
* <name>:<key/signature-in-Base64>
* ```
*/
struct BorrowedCryptoValue {
std::string_view name;
std::string_view payload;
/**
* This splits on the colon, the user can then separated decode the
* Base64 payload separately.
*/
static BorrowedCryptoValue parse(std::string_view);
};
struct Key
{
std::string name;
std::string key;
/**
* Construct Key from a string in the format
* ‘<name>:<key-in-base64>’.
*/
Key(std::string_view s);
std::string to_string() const;
protected:
Key(std::string_view name, std::string && key)
: name(name), key(std::move(key)) { }
};
struct PublicKey;
struct SecretKey : Key
{
SecretKey(std::string_view s);
/**
* Return a detached signature of the given string.
*/
std::string signDetached(std::string_view s) const;
PublicKey toPublicKey() const;
static SecretKey generate(std::string_view name);
private:
SecretKey(std::string_view name, std::string && key)
: Key(name, std::move(key)) { }
};
struct PublicKey : Key
{
PublicKey(std::string_view data);
/**
* @return true iff `sig` and this key's names match, and `sig` is a
* correct signature over `data` using the given public key.
*/
bool verifyDetached(std::string_view data, std::string_view sigs) const;
/**
* @return true iff `sig` is a correct signature over `data` using the
* given public key.
*
* @param just the Base64 signature itself, not a colon-separated pair of a
* public key name and signature.
*/
bool verifyDetachedAnon(std::string_view data, std::string_view sigs) const;
private:
PublicKey(std::string_view name, std::string && key)
: Key(name, std::move(key)) { }
friend struct SecretKey;
};
/**
* Map from key names to public keys
*/
typedef std::map<std::string, PublicKey> PublicKeys;
/**
* @return true iff ‘sig’ is a correct signature over ‘data’ using one
* of the given public keys.
*/
bool verifyDetached(std::string_view data, std::string_view sig, const PublicKeys & publicKeys);
}

23
src/libutil/signature/signer.cc Normal file
View file

@ -0,0 +1,23 @@
#include "signature/signer.hh"
#include "error.hh"
#include <sodium.h>
namespace nix {
LocalSigner::LocalSigner(SecretKey && privateKey)
: privateKey(privateKey)
, publicKey(privateKey.toPublicKey())
{ }
std::string LocalSigner::signDetached(std::string_view s) const
{
return privateKey.signDetached(s);
}
const PublicKey & LocalSigner::getPublicKey()
{
return publicKey;
}
}

61
src/libutil/signature/signer.hh Normal file
View file

@ -0,0 +1,61 @@
#pragma once
#include "types.hh"
#include "signature/local-keys.hh"
#include <map>
#include <optional>
namespace nix {
/**
* An abstract signer
*
* Derive from this class to implement a custom signature scheme.
*
* It is only necessary to implement signature of bytes and provide a
* public key.
*/
struct Signer
{
virtual ~Signer() = default;
/**
* Sign the given data, creating a (detached) signature.
*
* @param data data to be signed.
*
* @return the [detached
* signature](https://en.wikipedia.org/wiki/Detached_signature),
* i.e. just the signature itself without a copy of the signed data.
*/
virtual std::string signDetached(std::string_view data) const = 0;
/**
* View the public key associated with this `Signer`.
*/
virtual const PublicKey & getPublicKey() = 0;
};
using Signers = std::map<std::string, Signer*>;
/**
* Local signer
*
* The private key is held in this machine's RAM
*/
struct LocalSigner : Signer
{
LocalSigner(SecretKey && privateKey);
std::string signDetached(std::string_view s) const override;
const PublicKey & getPublicKey() override;
private:
SecretKey privateKey;
PublicKey publicKey;
};
}

105
src/libutil/source-path.cc Normal file
View file

@ -0,0 +1,105 @@
#include "source-path.hh"
namespace nix {
std::string_view SourcePath::baseName() const
{ return path.baseName().value_or("source"); }
SourcePath SourcePath::parent() const
{
auto p = path.parent();
assert(p);
return {accessor, std::move(*p)};
}
std::string SourcePath::readFile() const
{ return accessor->readFile(path); }
bool SourcePath::pathExists() const
{ return accessor->pathExists(path); }
InputAccessor::Stat SourcePath::lstat() const
{ return accessor->lstat(path); }
std::optional<InputAccessor::Stat> SourcePath::maybeLstat() const
{ return accessor->maybeLstat(path); }
InputAccessor::DirEntries SourcePath::readDirectory() const
{ return accessor->readDirectory(path); }
std::string SourcePath::readLink() const
{ return accessor->readLink(path); }
void SourcePath::dumpPath(
Sink & sink,
PathFilter & filter) const
{ return accessor->dumpPath(path, sink, filter); }
std::optional<CanonPath> SourcePath::getPhysicalPath() const
{ return accessor->getPhysicalPath(path); }
std::string SourcePath::to_string() const
{ return accessor->showPath(path); }
SourcePath SourcePath::operator+(const CanonPath & x) const
{ return {accessor, path + x}; }
SourcePath SourcePath::operator+(std::string_view c) const
{ return {accessor, path + c}; }
bool SourcePath::operator==(const SourcePath & x) const
{
return std::tie(*accessor, path) == std::tie(*x.accessor, x.path);
}
bool SourcePath::operator!=(const SourcePath & x) const
{
return std::tie(*accessor, path) != std::tie(*x.accessor, x.path);
}
bool SourcePath::operator<(const SourcePath & x) const
{
return std::tie(*accessor, path) < std::tie(*x.accessor, x.path);
}
SourcePath SourcePath::resolveSymlinks() const
{
auto res = SourcePath(accessor);
int linksAllowed = 1024;
std::list<std::string> todo;
for (auto & c : path)
todo.push_back(std::string(c));
while (!todo.empty()) {
auto c = *todo.begin();
todo.pop_front();
if (c == "" || c == ".")
;
else if (c == "..")
res.path.pop();
else {
res.path.push(c);
if (auto st = res.maybeLstat(); st && st->type == InputAccessor::tSymlink) {
if (!linksAllowed--)
throw Error("infinite symlink recursion in path '%s'", path);
auto target = res.readLink();
res.path.pop();
if (hasPrefix(target, "/"))
res.path = CanonPath::root;
todo.splice(todo.begin(), tokenizeString<std::list<std::string>>(target, "/"));
}
}
}
return res;
}
std::ostream & operator<<(std::ostream & str, const SourcePath & path)
{
str << path.to_string();
return str;
}
}

114
src/libutil/source-path.hh Normal file
View file

@ -0,0 +1,114 @@
#pragma once
/**
* @file
*
* @brief SourcePath
*/
#include "ref.hh"
#include "canon-path.hh"
#include "input-accessor.hh"
namespace nix {
/**
* An abstraction for accessing source files during
* evaluation. Currently, it's just a wrapper around `CanonPath` that
* accesses files in the regular filesystem, but in the future it will
* support fetching files in other ways.
*/
struct SourcePath
{
ref<InputAccessor> accessor;
CanonPath path;
SourcePath(ref<InputAccessor> accessor, CanonPath path = CanonPath::root)
: accessor(std::move(accessor))
, path(std::move(path))
{ }
std::string_view baseName() const;
/**
* Construct the parent of this `SourcePath`. Aborts if `this`
* denotes the root.
*/
SourcePath parent() const;
/**
* If this `SourcePath` denotes a regular file (not a symlink),
* return its contents; otherwise throw an error.
*/
std::string readFile() const;
/**
* Return whether this `SourcePath` denotes a file (of any type)
* that exists
*/
bool pathExists() const;
/**
* Return stats about this `SourcePath`, or throw an exception if
* it doesn't exist.
*/
InputAccessor::Stat lstat() const;
/**
* Return stats about this `SourcePath`, or std::nullopt if it
* doesn't exist.
*/
std::optional<InputAccessor::Stat> maybeLstat() const;
/**
* If this `SourcePath` denotes a directory (not a symlink),
* return its directory entries; otherwise throw an error.
*/
InputAccessor::DirEntries readDirectory() const;
/**
* If this `SourcePath` denotes a symlink, return its target;
* otherwise throw an error.
*/
std::string readLink() const;
/**
* Dump this `SourcePath` to `sink` as a NAR archive.
*/
void dumpPath(
Sink & sink,
PathFilter & filter = defaultPathFilter) const;
/**
* Return the location of this path in the "real" filesystem, if
* it has a physical location.
*/
std::optional<CanonPath> getPhysicalPath() const;
std::string to_string() const;
/**
* Append a `CanonPath` to this path.
*/
SourcePath operator + (const CanonPath & x) const;
/**
* Append a single component `c` to this path. `c` must not
* contain a slash. A slash is implicitly added between this path
* and `c`.
*/
SourcePath operator+(std::string_view c) const;
bool operator==(const SourcePath & x) const;
bool operator!=(const SourcePath & x) const;
bool operator<(const SourcePath & x) const;
/**
* Resolve any symlinks in this `SourcePath` (including its
* parents). The result is a `SourcePath` in which no element is a
* symlink.
*/
SourcePath resolveSymlinks() const;
};
std::ostream & operator << (std::ostream & str, const SourcePath & path);
}

2
src/libutil/thread-pool.cc
View file

@ -79,6 +79,8 @@ void ThreadPool::process()
void ThreadPool::doWork(bool mainThread)
{
ReceiveInterrupts receiveInterrupts;
if (!mainThread)
interruptCheck = [&]() { return (bool) quit; };

103
src/libutil/unix-domain-socket.cc
View file

@ -1,6 +1,7 @@
#include "file-system.hh"
#include "processes.hh"
#include "unix-domain-socket.hh"
#include "util.hh"
#include <sys/socket.h>
#include <sys/un.h>
@ -38,63 +39,69 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
}
static void bindConnectProcHelper(
std::string_view operationName, auto && operation,
int fd, const std::string & path)
{
struct sockaddr_un addr;
addr.sun_family = AF_UNIX;
// Casting between types like these legacy C library interfaces
// require is forbidden in C++. To maintain backwards
// compatibility, the implementation of the bind/connect functions
// contains some hints to the compiler that allow for this
// special case.
auto * psaddr = reinterpret_cast<struct sockaddr *>(&addr);
if (path.size() + 1 >= sizeof(addr.sun_path)) {
Pipe pipe;
pipe.create();
Pid pid = startProcess([&] {
try {
pipe.readSide.close();
Path dir = dirOf(path);
if (chdir(dir.c_str()) == -1)
throw SysError("chdir to '%s' failed", dir);
std::string base(baseNameOf(path));
if (base.size() + 1 >= sizeof(addr.sun_path))
throw Error("socket path '%s' is too long", base);
memcpy(addr.sun_path, base.c_str(), base.size() + 1);
if (operation(fd, psaddr, sizeof(addr)) == -1)
throw SysError("cannot %s to socket at '%s'", operationName, path);
writeFull(pipe.writeSide.get(), "0\n");
} catch (SysError & e) {
writeFull(pipe.writeSide.get(), fmt("%d\n", e.errNo));
} catch (...) {
writeFull(pipe.writeSide.get(), "-1\n");
}
});
pipe.writeSide.close();
auto errNo = string2Int<int>(chomp(drainFD(pipe.readSide.get())));
if (!errNo || *errNo == -1)
throw Error("cannot %s to socket at '%s'", operationName, path);
else if (*errNo > 0) {
errno = *errNo;
throw SysError("cannot %s to socket at '%s'", operationName, path);
}
} else {
memcpy(addr.sun_path, path.c_str(), path.size() + 1);
if (operation(fd, psaddr, sizeof(addr)) == -1)
throw SysError("cannot %s to socket at '%s'", operationName, path);
}
}
void bind(int fd, const std::string & path)
{
unlink(path.c_str());
struct sockaddr_un addr;
addr.sun_family = AF_UNIX;
if (path.size() + 1 >= sizeof(addr.sun_path)) {
Pid pid = startProcess([&]() {
Path dir = dirOf(path);
if (chdir(dir.c_str()) == -1)
throw SysError("chdir to '%s' failed", dir);
std::string base(baseNameOf(path));
if (base.size() + 1 >= sizeof(addr.sun_path))
throw Error("socket path '%s' is too long", base);
memcpy(addr.sun_path, base.c_str(), base.size() + 1);
if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot bind to socket '%s'", path);
_exit(0);
});
int status = pid.wait();
if (status != 0)
throw Error("cannot bind to socket '%s'", path);
} else {
memcpy(addr.sun_path, path.c_str(), path.size() + 1);
if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot bind to socket '%s'", path);
}
bindConnectProcHelper("bind", ::bind, fd, path);
}
void connect(int fd, const std::string & path)
{
struct sockaddr_un addr;
addr.sun_family = AF_UNIX;
if (path.size() + 1 >= sizeof(addr.sun_path)) {
Pid pid = startProcess([&]() {
Path dir = dirOf(path);
if (chdir(dir.c_str()) == -1)
throw SysError("chdir to '%s' failed", dir);
std::string base(baseNameOf(path));
if (base.size() + 1 >= sizeof(addr.sun_path))
throw Error("socket path '%s' is too long", base);
memcpy(addr.sun_path, base.c_str(), base.size() + 1);
if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot connect to socket at '%s'", path);
_exit(0);
});
int status = pid.wait();
if (status != 0)
throw Error("cannot connect to socket at '%s'", path);
} else {
memcpy(addr.sun_path, path.c_str(), path.size() + 1);
if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
throw SysError("cannot connect to socket at '%s'", path);
}
bindConnectProcHelper("connect", ::connect, fd, path);
}
}

6
src/libutil/url-parts.hh
View file

@ -8,7 +8,7 @@ namespace nix {
// URI stuff.
const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)";
const static std::string schemeNameRegex = "(?:[a-z][a-z0-9+.-]*)";
const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+(?:%\\w+)?";
const static std::string ipv6AddressRegex = "(?:\\[" + ipv6AddressSegmentRegex + "\\]|" + ipv6AddressSegmentRegex + ")";
const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
@ -19,13 +19,15 @@ const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncod
const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?";
const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])";
const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*";
const static std::string fragmentRegex = "(?:" + pcharRegex + "|[/? \"^])*";
const static std::string segmentRegex = "(?:" + pcharRegex + "*)";
const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)";
const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)";
/// A Git ref (i.e. branch or tag name).
/// \todo check that this is correct.
const static std::string refRegexS = "[a-zA-Z0-9@][a-zA-Z0-9_.\\/@-]*";
/// This regex incomplete. See https://git-scm.com/docs/git-check-ref-format
const static std::string refRegexS = "[a-zA-Z0-9@][a-zA-Z0-9_.\\/@+-]*";
extern std::regex refRegex;
/// Instead of defining what a good Git Ref is, we define what a bad Git Ref is

12
src/libutil/url.cc
View file

@ -13,10 +13,10 @@ std::regex revRegex(revRegexS, std::regex::ECMAScript);
ParsedURL parseURL(const std::string & url)
{
static std::regex uriRegex(
"((" + schemeRegex + "):"
"((" + schemeNameRegex + "):"
+ "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))"
+ "(?:\\?(" + queryRegex + "))?"
+ "(?:#(" + queryRegex + "))?",
+ "(?:#(" + fragmentRegex + "))?",
std::regex::ECMAScript);
std::smatch match;
@ -183,4 +183,12 @@ std::string fixGitURL(const std::string & url)
}
}
// https://www.rfc-editor.org/rfc/rfc3986#section-3.1
bool isValidSchemeName(std::string_view s)
{
static std::regex regex(schemeNameRegex, std::regex::ECMAScript);
return std::regex_match(s.begin(), s.end(), regex, std::regex_constants::match_default);
}
}

9
src/libutil/url.hh
View file

@ -55,4 +55,13 @@ ParsedUrlScheme parseUrlScheme(std::string_view scheme);
changes absolute paths into file:// URLs. */
std::string fixGitURL(const std::string & url);
/**
* Whether a string is valid as RFC 3986 scheme name.
* Colon `:` is part of the URI; not the scheme name, and therefore rejected.
* See https://www.rfc-editor.org/rfc/rfc3986#section-3.1
*
* Does not check whether the scheme is understood, as that's context-dependent.
*/
bool isValidSchemeName(std::string_view scheme);
}

27
src/libutil/util.cc
View file

@ -7,6 +7,7 @@
#include <grp.h>
#include <regex>
#include <sodium.h>
namespace nix {
@ -19,7 +20,7 @@ void initLibUtil() {
// When exception handling fails, the message tends to be printed by the
// C++ runtime, followed by an abort.
// For example on macOS we might see an error such as
// libc++abi: terminating with uncaught exception of type nix::SysError: error: C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded.
// libc++abi: terminating with uncaught exception of type nix::SystemError: error: C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded.
bool caught = false;
try {
throwExceptionSelfCheck();
@ -28,6 +29,9 @@ void initLibUtil() {
}
// This is not actually the main point of this check, but let's make sure anyway:
assert(caught);
if (sodium_init() == -1)
throw Error("could not initialise libsodium");
}
//////////////////////////////////////////////////////////////////////
@ -48,9 +52,9 @@ template<class C> C tokenizeString(std::string_view s, std::string_view separato
{
C result;
auto pos = s.find_first_not_of(separators, 0);
while (pos != std::string_view::npos) {
while (pos != s.npos) {
auto end = s.find_first_of(separators, pos + 1);
if (end == std::string_view::npos) end = s.size();
if (end == s.npos) end = s.size();
result.insert(result.end(), std::string(s, pos, end - pos));
pos = s.find_first_not_of(separators, end);
}
@ -65,7 +69,7 @@ template std::vector<std::string> tokenizeString(std::string_view s, std::string
std::string chomp(std::string_view s)
{
size_t i = s.find_last_not_of(" \n\r\t");
return i == std::string_view::npos ? "" : std::string(s, 0, i + 1);
return i == s.npos ? "" : std::string(s, 0, i + 1);
}
@ -85,7 +89,7 @@ std::string replaceStrings(
{
if (from.empty()) return res;
size_t pos = 0;
while ((pos = res.find(from, pos)) != std::string::npos) {
while ((pos = res.find(from, pos)) != res.npos) {
res.replace(pos, from.size(), to);
pos += to.size();
}
@ -98,7 +102,7 @@ std::string rewriteStrings(std::string s, const StringMap & rewrites)
for (auto & i : rewrites) {
if (i.first == i.second) continue;
size_t j = 0;
while ((j = s.find(i.first, j)) != std::string::npos)
while ((j = s.find(i.first, j)) != s.npos)
s.replace(j, i.first.size(), i.second);
}
return s;
@ -118,12 +122,11 @@ bool hasSuffix(std::string_view s, std::string_view suffix)
}
std::string toLower(const std::string & s)
std::string toLower(std::string s)
{
std::string r(s);
for (auto & c : r)
for (auto & c : s)
c = std::tolower(c);
return r;
return s;
}
@ -131,7 +134,7 @@ std::string shellEscape(const std::string_view s)
{
std::string r;
r.reserve(s.size() + 2);
r += "'";
r += '\'';
for (auto & i : s)
if (i == '\'') r += "'\\''"; else r += i;
r += '\'';
@ -180,7 +183,7 @@ std::string base64Encode(std::string_view s)
std::string base64Decode(std::string_view s)
{
constexpr char npos = -1;
constexpr std::array<char, 256> base64DecodeChars = [&]() {
constexpr std::array<char, 256> base64DecodeChars = [&] {
std::array<char, 256> result{};
for (auto& c : result)
c = npos;

2
src/libutil/util.hh
View file

@ -180,7 +180,7 @@ bool hasSuffix(std::string_view s, std::string_view suffix);
/**
* Convert a string to lower case.
*/
std::string toLower(const std::string & s);
std::string toLower(std::string s);
/**