1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-29 06:21:14 +02:00

Merge remote-tracking branch 'origin/master' into flakes

This commit is contained in:
Eelco Dolstra 2019-12-11 14:53:30 +01:00
commit ecb3a1afa2
119 changed files with 3905 additions and 2250 deletions

View file

@ -1,5 +1,6 @@
#pragma once
#include "path.hh"
#include "hash.hh"
#include "serialise.hh"
#include "crypto.hh"
@ -43,14 +44,11 @@ enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };
/* Size of the hash part of store paths, in base-32 characters. */
const size_t storePathHashLen = 32; // i.e. 160 bits
/* Magic header of exportPath() output (obsolete). */
const uint32_t exportMagic = 0x4558494e;
typedef std::unordered_map<Path, std::unordered_set<std::string>> Roots;
typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots;
struct GCOptions
@ -84,7 +82,7 @@ struct GCOptions
bool ignoreLiveness{false};
/* For `gcDeleteSpecific', the paths to delete. */
PathSet pathsToDelete;
StorePathSet pathsToDelete;
/* Stop after at least `maxFreed' bytes have been freed. */
unsigned long long maxFreed{std::numeric_limits<unsigned long long>::max()};
@ -105,21 +103,21 @@ struct GCResults
struct SubstitutablePathInfo
{
Path deriver;
PathSet references;
std::optional<StorePath> deriver;
StorePathSet references;
unsigned long long downloadSize; /* 0 = unknown or inapplicable */
unsigned long long narSize; /* 0 = unknown */
};
typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos;
typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
struct ValidPathInfo
{
Path path;
Path deriver;
StorePath path;
std::optional<StorePath> deriver;
Hash narHash;
PathSet references;
StorePathSet references;
time_t registrationTime = 0;
uint64_t narSize = 0; // 0 = unknown
uint64_t id; // internal use only
@ -144,7 +142,7 @@ struct ValidPathInfo
Ideally, the content-addressability assertion would just be a
Boolean, and the store path would be computed from
storePathToName(path), narHash and references. However,
the name component, narHash and references. However,
1) we've accumulated several types of content-addressed paths
over the years; and 2) fixed-output derivations support
multiple hash algorithms and serialisation methods (flat file
@ -172,9 +170,9 @@ struct ValidPathInfo
the NAR, and the sorted references. The size field is strictly
speaking superfluous, but might prevent endless/excessive data
attacks. */
std::string fingerprint() const;
std::string fingerprint(const Store & store) const;
void sign(const SecretKey & secretKey);
void sign(const Store & store, const SecretKey & secretKey);
/* Return true iff the path is verifiably content-addressed. */
bool isContentAddressed(const Store & store) const;
@ -187,10 +185,13 @@ struct ValidPathInfo
size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const;
/* Verify a single signature. */
bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const;
Strings shortRefs() const;
ValidPathInfo(StorePath && path) : path(std::move(path)) { }
explicit ValidPathInfo(const ValidPathInfo & other);
virtual ~ValidPathInfo() { }
};
@ -241,6 +242,23 @@ struct BuildResult
};
struct StorePathWithOutputs
{
StorePath path;
std::set<std::string> outputs;
StorePathWithOutputs(const StorePath & path, const std::set<std::string> & outputs = {})
: path(path.clone()), outputs(outputs)
{ }
StorePathWithOutputs(const StorePathWithOutputs & other)
: path(other.path.clone()), outputs(other.outputs)
{ }
std::string to_string(const Store & store) const;
};
class Store : public std::enable_shared_from_this<Store>, public Config
{
public:
@ -259,6 +277,7 @@ protected:
struct State
{
// FIXME: fix key
LRUCache<std::string, std::shared_ptr<const ValidPathInfo>> pathInfoCache;
};
@ -274,6 +293,24 @@ public:
virtual std::string getUri() = 0;
StorePath parseStorePath(std::string_view path) const;
std::string printStorePath(const StorePath & path) const;
// FIXME: remove
StorePathSet parseStorePathSet(const PathSet & paths) const;
PathSet printStorePathSet(const StorePathSet & path) const;
/* Split a string specifying a derivation and a set of outputs
(/nix/store/hash-foo!out1,out2,...) into the derivation path
and the outputs. */
StorePathWithOutputs parseDrvPathWithOutputs(const string & s);
/* Display a set of paths in human-readable form (i.e., between quotes
and separated by commas). */
std::string showPaths(const StorePathSet & paths);
/* Return true if path is in the Nix store (but not the Nix
store itself). */
bool isInStore(const Path & path) const;
@ -282,9 +319,6 @@ public:
the Nix store. */
bool isStorePath(const Path & path) const;
/* Throw an exception if path is not a store path. */
void assertStorePath(const Path & path) const;
/* Chop off the parts after the top-level store name, e.g.,
/nix/store/abcd-foo/bar => /nix/store/abcd-foo. */
Path toStorePath(const Path & path) const;
@ -294,26 +328,27 @@ public:
/* Same as followLinksToStore(), but apply toStorePath() to the
result. */
Path followLinksToStorePath(const Path & path) const;
StorePath followLinksToStorePath(const Path & path) const;
/* Constructs a unique store path name. */
Path makeStorePath(const string & type,
const Hash & hash, const string & name) const;
StorePath makeStorePath(const string & type,
const Hash & hash, std::string_view name) const;
Path makeOutputPath(const string & id,
const Hash & hash, const string & name) const;
StorePath makeOutputPath(const string & id,
const Hash & hash, std::string_view name) const;
Path makeFixedOutputPath(bool recursive,
const Hash & hash, const string & name,
const PathSet & references = {}) const;
StorePath makeFixedOutputPath(bool recursive,
const Hash & hash, std::string_view name,
const StorePathSet & references = {},
bool hasSelfReference = false) const;
Path makeTextPath(const string & name, const Hash & hash,
const PathSet & references) const;
StorePath makeTextPath(std::string_view name, const Hash & hash,
const StorePathSet & references) const;
/* This is the preparatory part of addToStore(); it computes the
store path to which srcPath is to be copied. Returns the store
path and the cryptographic hash of the contents of srcPath. */
std::pair<Path, Hash> computeStorePathForPath(const string & name,
std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name,
const Path & srcPath, bool recursive = true,
HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const;
@ -331,21 +366,21 @@ public:
simply yield a different store path, so other users wouldn't be
affected), but it has some backwards compatibility issues (the
hashing scheme changes), so I'm not doing that for now. */
Path computeStorePathForText(const string & name, const string & s,
const PathSet & references) const;
StorePath computeStorePathForText(const string & name, const string & s,
const StorePathSet & references) const;
/* Check whether a path is valid. */
bool isValidPath(const Path & path);
bool isValidPath(const StorePath & path);
protected:
virtual bool isValidPathUncached(const Path & path);
virtual bool isValidPathUncached(const StorePath & path);
public:
/* Query which of the given paths is valid. Optionally, try to
substitute missing paths. */
virtual PathSet queryValidPaths(const PathSet & paths,
virtual StorePathSet queryValidPaths(const StorePathSet & paths,
SubstituteFlag maybeSubstitute = NoSubstitute);
/* Query the set of all valid paths. Note that for some store
@ -353,54 +388,54 @@ public:
(i.e. you'll get /nix/store/<hash> rather than
/nix/store/<hash>-<name>). Use queryPathInfo() to obtain the
full store path. */
virtual PathSet queryAllValidPaths()
virtual StorePathSet queryAllValidPaths()
{ unsupported("queryAllValidPaths"); }
/* Query information about a valid path. It is permitted to omit
the name part of the store path. */
ref<const ValidPathInfo> queryPathInfo(const Path & path);
ref<const ValidPathInfo> queryPathInfo(const StorePath & path);
/* Asynchronous version of queryPathInfo(). */
void queryPathInfo(const Path & path,
void queryPathInfo(const StorePath & path,
Callback<ref<const ValidPathInfo>> callback) noexcept;
protected:
virtual void queryPathInfoUncached(const Path & path,
virtual void queryPathInfoUncached(const StorePath & path,
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept = 0;
public:
/* Queries the set of incoming FS references for a store path.
The result is not cleared. */
virtual void queryReferrers(const Path & path, PathSet & referrers)
virtual void queryReferrers(const StorePath & path, StorePathSet & referrers)
{ unsupported("queryReferrers"); }
/* Return all currently valid derivations that have `path' as an
output. (Note that the result of `queryDeriver()' is the
derivation that was actually used to produce `path', which may
not exist anymore.) */
virtual PathSet queryValidDerivers(const Path & path) { return {}; };
virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; };
/* Query the outputs of the derivation denoted by `path'. */
virtual PathSet queryDerivationOutputs(const Path & path)
virtual StorePathSet queryDerivationOutputs(const StorePath & path)
{ unsupported("queryDerivationOutputs"); }
/* Query the output names of the derivation denoted by `path'. */
virtual StringSet queryDerivationOutputNames(const Path & path)
virtual StringSet queryDerivationOutputNames(const StorePath & path)
{ unsupported("queryDerivationOutputNames"); }
/* Query the full store path given the hash part of a valid store
path, or "" if the path doesn't exist. */
virtual Path queryPathFromHashPart(const string & hashPart) = 0;
path, or empty if the path doesn't exist. */
virtual std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) = 0;
/* Query which of the given paths have substitutes. */
virtual PathSet querySubstitutablePaths(const PathSet & paths) { return {}; };
virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; };
/* Query substitute info (i.e. references, derivers and download
sizes) of a set of paths. If a path does not have substitute
info, it's omitted from the resulting infos map. */
virtual void querySubstitutablePathInfos(const PathSet & paths,
virtual void querySubstitutablePathInfos(const StorePathSet & paths,
SubstitutablePathInfos & infos) { return; };
virtual bool wantMassQuery() { return false; }
@ -419,12 +454,12 @@ public:
validity the resulting path. The resulting path is returned.
The function object `filter' can be used to exclude files (see
libutil/archive.hh). */
virtual Path addToStore(const string & name, const Path & srcPath,
virtual StorePath addToStore(const string & name, const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
// FIXME: remove?
virtual Path addToStoreFromDump(const string & dump, const string & name,
virtual StorePath addToStoreFromDump(const string & dump, const string & name,
bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
{
throw Error("addToStoreFromDump() is not supported by this store");
@ -432,11 +467,11 @@ public:
/* Like addToStore, but the contents written to the output path is
a regular file containing the given string. */
virtual Path addTextToStore(const string & name, const string & s,
const PathSet & references, RepairFlag repair = NoRepair) = 0;
virtual StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair = NoRepair) = 0;
/* Write a NAR dump of a store path. */
virtual void narFromPath(const Path & path, Sink & sink) = 0;
virtual void narFromPath(const StorePath & path, Sink & sink) = 0;
/* For each path, if it's a derivation, build it. Building a
derivation means ensuring that the output paths are valid. If
@ -446,22 +481,24 @@ public:
output paths can be created by running the builder, after
recursively building any sub-derivations. For inputs that are
not derivations, substitute them. */
virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal);
virtual void buildPaths(
const std::vector<StorePathWithOutputs> & paths,
BuildMode buildMode = bmNormal);
/* Build a single non-materialized derivation (i.e. not from an
on-disk .drv file). Note that drvPath is only used for
informational purposes. */
virtual BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode = bmNormal) = 0;
/* Ensure that a path is valid. If it is not currently valid, it
may be made valid by running a substitute (if defined for the
path). */
virtual void ensurePath(const Path & path) = 0;
virtual void ensurePath(const StorePath & path) = 0;
/* Add a store path as a temporary root of the garbage collector.
The root disappears as soon as we exit. */
virtual void addTempRoot(const Path & path)
virtual void addTempRoot(const StorePath & path)
{ unsupported("addTempRoot"); }
/* Add an indirect root, which is merely a symlink to `path' from
@ -507,7 +544,7 @@ public:
/* Return a string representing information about the path that
can be loaded into the database using `nix-store --load-db' or
`nix-store --register-validity'. */
string makeValidityRegistration(const PathSet & paths,
string makeValidityRegistration(const StorePathSet & paths,
bool showDerivers, bool showHash);
/* Write a JSON representation of store path metadata, such as the
@ -515,14 +552,14 @@ public:
variable elements such as the registration time are
included. If showClosureSize is true, the closure size of
each path is included. */
void pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths,
void pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths,
bool includeImpureInfo, bool showClosureSize,
AllowInvalidFlag allowInvalid = DisallowInvalid);
/* Return the size of the closure of the specified path, that is,
the sum of the size of the NAR serialisation of each path in
the closure. */
std::pair<uint64_t, uint64_t> getClosureSize(const Path & storePath);
std::pair<uint64_t, uint64_t> getClosureSize(const StorePath & storePath);
/* Optimise the disk space usage of the Nix store by hard-linking files
with the same contents. */
@ -538,14 +575,14 @@ public:
/* Add signatures to the specified store path. The signatures are
not verified. */
virtual void addSignatures(const Path & storePath, const StringSet & sigs)
virtual void addSignatures(const StorePath & storePath, const StringSet & sigs)
{ unsupported("addSignatures"); }
/* Utility functions. */
/* Read a derivation, after ensuring its existence through
ensurePath(). */
Derivation derivationFromPath(const Path & drvPath);
Derivation derivationFromPath(const StorePath & drvPath);
/* Place in `out' the set of all store paths in the file system
closure of `storePath'; that is, all paths than can be directly
@ -554,36 +591,36 @@ public:
`storePath' is returned; that is, the closures under the
`referrers' relation instead of the `references' relation is
returned. */
virtual void computeFSClosure(const PathSet & paths,
PathSet & out, bool flipDirection = false,
virtual void computeFSClosure(const StorePathSet & paths,
StorePathSet & out, bool flipDirection = false,
bool includeOutputs = false, bool includeDerivers = false);
void computeFSClosure(const Path & path,
PathSet & out, bool flipDirection = false,
void computeFSClosure(const StorePath & path,
StorePathSet & out, bool flipDirection = false,
bool includeOutputs = false, bool includeDerivers = false);
/* Given a set of paths that are to be built, return the set of
derivations that will be built, and the set of output paths
that will be substituted. */
virtual void queryMissing(const PathSet & targets,
PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown,
virtual void queryMissing(const std::vector<StorePathWithOutputs> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
unsigned long long & downloadSize, unsigned long long & narSize);
/* Sort a set of paths topologically under the references
relation. If p refers to q, then p precedes q in this list. */
Paths topoSortPaths(const PathSet & paths);
StorePaths topoSortPaths(const StorePathSet & paths);
/* Export multiple paths in the format expected by nix-store
--import. */
void exportPaths(const Paths & paths, Sink & sink);
void exportPaths(const StorePathSet & paths, Sink & sink);
void exportPath(const Path & path, Sink & sink);
void exportPath(const StorePath & path, Sink & sink);
/* Import a sequence of NAR dumps created by exportPaths() into
the Nix store. Optionally, the contents of the NARs are
preloaded into the specified FS accessor to speed up subsequent
access. */
Paths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
StorePaths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
CheckSigsFlag checkSigs = CheckSigs);
struct Stats
@ -607,7 +644,7 @@ public:
/* Return the build log of the specified store path, if available,
or null otherwise. */
virtual std::shared_ptr<std::string> getBuildLog(const Path & path)
virtual std::shared_ptr<std::string> getBuildLog(const StorePath & path)
{ return nullptr; }
/* Hack to allow long-running processes like hydra-queue-runner to
@ -673,11 +710,11 @@ public:
LocalFSStore(const Params & params);
void narFromPath(const Path & path, Sink & sink) override;
void narFromPath(const StorePath & path, Sink & sink) override;
ref<FSAccessor> getFSAccessor() override;
/* Register a permanent GC root. */
Path addPermRoot(const Path & storePath,
Path addPermRoot(const StorePath & storePath,
const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false);
virtual Path getRealStoreDir() { return storeDir; }
@ -688,25 +725,17 @@ public:
return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1);
}
std::shared_ptr<std::string> getBuildLog(const Path & path) override;
std::shared_ptr<std::string> getBuildLog(const StorePath & path) override;
};
/* Extract the name part of the given store path. */
string storePathToName(const Path & path);
/* Extract the hash part of the given store path. */
string storePathToHash(const Path & path);
/* Check whether name is a valid store path name part, i.e. contains
only the characters [a-zA-Z0-9\+\-\.\_\?\=] and doesn't start with
a dot. */
void checkStoreName(const string & name);
/* Copy a path from one store to another. */
void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
const Path & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs);
const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs);
/* Copy store paths from one store to another. The paths may be copied
@ -714,7 +743,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
(i.e. if A is a reference of B, then A is copied before B), but
the set of store paths is not automatically closed; use
copyClosure() for that. */
void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths,
void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute);
@ -722,7 +751,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa
/* Copy the closure of the specified paths from one store to another. */
void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
const PathSet & storePaths,
const StorePathSet & storePaths,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute);
@ -805,7 +834,9 @@ struct RegisterStoreImplementation
string showPaths(const PathSet & paths);
ValidPathInfo decodeValidPathInfo(std::istream & str,
std::optional<ValidPathInfo> decodeValidPathInfo(
const Store & store,
std::istream & str,
bool hashGiven = false);