Allow content-addressable paths to have references

This adds a command 'nix make-content-addressable' that rewrites the
specified store paths into content-addressable paths. The advantage of
such paths is that 1) they can be imported without signatures; 2) they
can enable deduplication in cases where derivation changes do not
cause output changes (apart from store path hashes).

For example,

  $ nix make-content-addressable -r nixpkgs.cowsay
  rewrote '/nix/store/g1g31ah55xdia1jdqabv1imf6mcw0nb1-glibc-2.25-49' to '/nix/store/48jfj7bg78a8n4f2nhg269rgw1936vj4-glibc-2.25-49'
  ...
  rewrote '/nix/store/qbi6rzpk0bxjw8lw6azn2mc7ynnn455q-cowsay-3.03+dfsg1-16' to '/nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16'

We can then copy the resulting closure to another store without
signatures:

  $ nix copy --trusted-public-keys '' ---to ~/my-nix /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16

In order to support self-references in content-addressable paths,
these paths are hashed "modulo" self-references, meaning that
self-references are zeroed out during hashing. Somewhat annoyingly,
this means that the NAR hash stored in the Nix database is no longer
necessarily equal to the output of "nix hash-path"; for
content-addressable paths, you need to pass the --modulo flag:

  $ nix path-info --json /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16  | jq -r .[].narHash
  sha256:0ri611gdilz2c9rsibqhsipbfs9vwcqvs811a52i2bnkhv7w9mgw

  $ nix hash-path --type sha256 --base32 /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16
  1ggznh07khq0hz6id09pqws3a8q9pn03ya3c03nwck1kwq8rclzs

  $ nix hash-path --type sha256 --base32 /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16 --modulo iq6g2x4q62xp7y7493bibx0qn5w7xz67
  0ri611gdilz2c9rsibqhsipbfs9vwcqvs811a52i2bnkhv7w9mgw

src/libutil/hash.cc

@@ -256,23 +256,9 @@ Hash hashString(HashType ht, const string & s)
 
 Hash hashFile(HashType ht, const Path & path)
 {
-    Ctx ctx;
-    Hash hash(ht);
-    start(ht, ctx);
-
-    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
-    if (!fd) throw SysError(format("opening file '%1%'") % path);
-
-    std::vector<unsigned char> buf(8192);
-    ssize_t n;
-    while ((n = read(fd.get(), buf.data(), buf.size()))) {
-        checkInterrupt();
-        if (n == -1) throw SysError(format("reading file '%1%'") % path);
-        update(ht, ctx, buf.data(), n);
-    }
-
-    finish(ht, ctx, hash.hash);
-    return hash;
+    HashSink sink(ht);
+    readFile(path, sink);
+    return sink.finish().first;
 }
 
 

src/libutil/hash.hh

@@ -111,7 +111,12 @@ string printHashType(HashType ht);
 
 union Ctx;
 
-class HashSink : public BufferedSink
+struct AbstractHashSink : virtual Sink
+{
+    virtual HashResult finish() = 0;
+};
+
+class HashSink : public BufferedSink, public AbstractHashSink
 {
 private:
     HashType ht;
@@ -122,8 +127,8 @@ public:
     HashSink(HashType ht);
     HashSink(const HashSink & h);
     ~HashSink();
-    void write(const unsigned char * data, size_t len);
-    HashResult finish();
+    void write(const unsigned char * data, size_t len) override;
+    HashResult finish() override;
     HashResult currentHash();
 };
 

src/libutil/serialise.hh

@@ -24,7 +24,7 @@ struct Sink
 
 
 /* A buffered abstract sink. */
-struct BufferedSink : Sink
+struct BufferedSink : virtual Sink
 {
     size_t bufSize, bufPos;
     std::unique_ptr<unsigned char[]> buffer;

src/libutil/util.cc

@@ -1260,6 +1260,19 @@ string replaceStrings(const std::string & s,
 }
 
 
+std::string rewriteStrings(const std::string & _s, const StringMap & rewrites)
+{
+    auto s = _s;
+    for (auto & i : rewrites) {
+        if (i.first == i.second) continue;
+        size_t j = 0;
+        while ((j = s.find(i.first, j)) != string::npos)
+            s.replace(j, i.first.size(), i.second);
+    }
+    return s;
+}
+
+
 string statusToString(int status)
 {
     if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {

src/libutil/util.hh

@@ -362,6 +362,20 @@ string replaceStrings(const std::string & s,
     const std::string & from, const std::string & to);
 
 
+std::string rewriteStrings(const std::string & s, const StringMap & rewrites);
+
+
+/* If a set contains 'from', remove it and insert 'to'. */
+template<typename T>
+void replaceInSet(std::set<T> & set, const T & from, const T & to)
+{
+    auto i = set.find(from);
+    if (i == set.end()) return;
+    set.erase(i);
+    set.insert(to);
+}
+
+
 /* Convert the exit status of a child as returned by wait() into an
    error string. */
 string statusToString(int status);