1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2025-06-28 17:51:15 +02:00

Create header for LegacySSHStore

In https://github.com/NixOS/nix/pull/6134#issuecomment-1079199888,
@thuffschmitt proposed exposing `LegacySSHStore` in Nix for
deduplication with Hydra, at least temporarily. I think that is a good
idea.

Note that the diff will look bad unless one ignores whitespace! Also try
this locally:

```shell-session
git diff --ignore-all-space HEAD^:src/libstore/legacy-ssh-store.cc HEAD:src/libstore/legacy-ssh-store.cc
git diff --ignore-all-space HEAD^:src/libstore/legacy-ssh-store.cc HEAD:src/libstore/legacy-ssh-store.hh
```
This commit is contained in:
John Ericson 2023-12-10 14:28:14 -05:00
parent b7e016ab24
commit deadb3bfe9
2 changed files with 466 additions and 392 deletions

View file

@ -1,3 +1,4 @@
#include "legacy-ssh-store.hh"
#include "ssh-store-config.hh" #include "ssh-store-config.hh"
#include "archive.hh" #include "archive.hh"
#include "pool.hh" #include "pool.hh"
@ -13,35 +14,16 @@
namespace nix { namespace nix {
struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig std::string LegacySSHStoreConfig::doc()
{ {
using CommonSSHStoreConfig::CommonSSHStoreConfig;
const Setting<Path> remoteProgram{this, "nix-store", "remote-program",
"Path to the `nix-store` executable on the remote machine."};
const Setting<int> maxConnections{this, 1, "max-connections",
"Maximum number of concurrent SSH connections."};
const std::string name() override { return "SSH Store"; }
std::string doc() override
{
return return
#include "legacy-ssh-store.md" #include "legacy-ssh-store.md"
; ;
} }
};
struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store
struct LegacySSHStore::Connection
{ {
// Hack for getting remote build log output.
// Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
// the documentation
const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
struct Connection
{
std::unique_ptr<SSHMaster::Connection> sshConn; std::unique_ptr<SSHMaster::Connection> sshConn;
FdSink to; FdSink to;
FdSource from; FdSource from;
@ -79,17 +61,10 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
.version = remoteVersion, .version = remoteVersion,
}; };
} }
}; };
std::string host;
ref<Pool<Connection>> connections; LegacySSHStore::LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params)
SSHMaster master;
static std::set<std::string> uriSchemes() { return {"ssh"}; }
LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params)
: StoreConfig(params) : StoreConfig(params)
, CommonSSHStoreConfig(params) , CommonSSHStoreConfig(params)
, LegacySSHStoreConfig(params) , LegacySSHStoreConfig(params)
@ -108,11 +83,12 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
connections->capacity() > 1, connections->capacity() > 1,
compress, compress,
logFD) logFD)
{ {
} }
ref<Connection> openConnection()
{ ref<LegacySSHStore::Connection> LegacySSHStore::openConnection()
{
auto conn = make_ref<Connection>(); auto conn = make_ref<Connection>();
conn->sshConn = master.startCommand( conn->sshConn = master.startCommand(
fmt("%s --serve --write", remoteProgram) fmt("%s --serve --write", remoteProgram)
@ -147,16 +123,18 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
} }
return conn; return conn;
}; };
std::string getUri() override
{ std::string LegacySSHStore::getUri()
{
return *uriSchemes().begin() + "://" + host; return *uriSchemes().begin() + "://" + host;
} }
void queryPathInfoUncached(const StorePath & path,
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override void LegacySSHStore::queryPathInfoUncached(const StorePath & path,
{ Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
{
try { try {
auto conn(connections->get()); auto conn(connections->get());
@ -184,11 +162,12 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
callback(std::move(info)); callback(std::move(info));
} catch (...) { callback.rethrow(); } } catch (...) { callback.rethrow(); }
} }
void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs) override void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source,
{ RepairFlag repair, CheckSigsFlag checkSigs)
{
debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host);
auto conn(connections->get()); auto conn(connections->get());
@ -240,41 +219,21 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
if (readInt(conn->from) != 1) if (readInt(conn->from) != 1)
throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host);
} }
void narFromPath(const StorePath & path, Sink & sink) override
{ void LegacySSHStore::narFromPath(const StorePath & path, Sink & sink)
{
auto conn(connections->get()); auto conn(connections->get());
conn->to << ServeProto::Command::DumpStorePath << printStorePath(path); conn->to << ServeProto::Command::DumpStorePath << printStorePath(path);
conn->to.flush(); conn->to.flush();
copyNAR(conn->from, sink); copyNAR(conn->from, sink);
} }
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
{ unsupported("queryPathFromHashPart"); }
StorePath addToStore( void LegacySSHStore::putBuildSettings(Connection & conn)
std::string_view name, {
const Path & srcPath,
FileIngestionMethod method,
HashAlgorithm hashAlgo,
PathFilter & filter,
RepairFlag repair,
const StorePathSet & references) override
{ unsupported("addToStore"); }
StorePath addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair) override
{ unsupported("addTextToStore"); }
private:
void putBuildSettings(Connection & conn)
{
ServeProto::write(*this, conn, ServeProto::BuildOptions { ServeProto::write(*this, conn, ServeProto::BuildOptions {
.maxSilentTime = settings.maxSilentTime, .maxSilentTime = settings.maxSilentTime,
.buildTimeout = settings.buildTimeout, .buildTimeout = settings.buildTimeout,
@ -283,13 +242,12 @@ private:
.enforceDeterminism = 0, .enforceDeterminism = 0,
.keepFailed = settings.keepFailed, .keepFailed = settings.keepFailed,
}); });
} }
public:
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildResult LegacySSHStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override BuildMode buildMode)
{ {
auto conn(connections->get()); auto conn(connections->get());
conn->to conn->to
@ -302,10 +260,11 @@ public:
conn->to.flush(); conn->to.flush();
return ServeProto::Serialise<BuildResult>::read(*this, *conn); return ServeProto::Serialise<BuildResult>::read(*this, *conn);
} }
void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override
{ void LegacySSHStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore)
{
if (evalStore && evalStore.get() != this) if (evalStore && evalStore.get() != this)
throw Error("building on an SSH store is incompatible with '--eval-store'"); throw Error("building on an SSH store is incompatible with '--eval-store'");
@ -340,29 +299,13 @@ public:
conn->from >> result.errorMsg; conn->from >> result.errorMsg;
throw Error(result.status, result.errorMsg); throw Error(result.status, result.errorMsg);
} }
} }
void ensurePath(const StorePath & path) override
{ unsupported("ensurePath"); }
virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override void LegacySSHStore::computeFSClosure(const StorePathSet & paths,
{ unsupported("getFSAccessor"); } StorePathSet & out, bool flipDirection,
bool includeOutputs, bool includeDerivers)
/** {
* The default instance would schedule the work on the client side, but
* for consistency with `buildPaths` and `buildDerivation` it should happen
* on the remote side.
*
* We make this fail for now so we can add implement this properly later
* without it being a breaking change.
*/
void repairPath(const StorePath & path) override
{ unsupported("repairPath"); }
void computeFSClosure(const StorePathSet & paths,
StorePathSet & out, bool flipDirection = false,
bool includeOutputs = false, bool includeDerivers = false) override
{
if (flipDirection || includeDerivers) { if (flipDirection || includeDerivers) {
Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers); Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers);
return; return;
@ -378,11 +321,12 @@ public:
for (auto & i : ServeProto::Serialise<StorePathSet>::read(*this, *conn)) for (auto & i : ServeProto::Serialise<StorePathSet>::read(*this, *conn))
out.insert(i); out.insert(i);
} }
StorePathSet queryValidPaths(const StorePathSet & paths,
SubstituteFlag maybeSubstitute = NoSubstitute) override StorePathSet LegacySSHStore::queryValidPaths(const StorePathSet & paths,
{ SubstituteFlag maybeSubstitute)
{
auto conn(connections->get()); auto conn(connections->get());
conn->to conn->to
@ -393,33 +337,31 @@ public:
conn->to.flush(); conn->to.flush();
return ServeProto::Serialise<StorePathSet>::read(*this, *conn); return ServeProto::Serialise<StorePathSet>::read(*this, *conn);
} }
void connect() override
{ void LegacySSHStore::connect()
{
auto conn(connections->get()); auto conn(connections->get());
} }
unsigned int getProtocol() override
{ unsigned int LegacySSHStore::getProtocol()
{
auto conn(connections->get()); auto conn(connections->get());
return conn->remoteVersion; return conn->remoteVersion;
} }
/**
/**
* The legacy ssh protocol doesn't support checking for trusted-user. * The legacy ssh protocol doesn't support checking for trusted-user.
* Try using ssh-ng:// instead if you want to know. * Try using ssh-ng:// instead if you want to know.
*/ */
std::optional<TrustedFlag> isTrustedClient() override std::optional<TrustedFlag> isTrustedClient()
{ {
return std::nullopt; return std::nullopt;
} }
void queryRealisationUncached(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override
// TODO: Implement
{ unsupported("queryRealisation"); }
};
static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regLegacySSHStore; static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regLegacySSHStore;

View file

@ -0,0 +1,132 @@
#pragma once
///@file
#include "ssh-store-config.hh"
#include "store-api.hh"
#include "ssh.hh"
#include "callback.hh"
#include "pool.hh"
namespace nix {
struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
{
using CommonSSHStoreConfig::CommonSSHStoreConfig;
const Setting<Path> remoteProgram{this, "nix-store", "remote-program",
"Path to the `nix-store` executable on the remote machine."};
const Setting<int> maxConnections{this, 1, "max-connections",
"Maximum number of concurrent SSH connections."};
const std::string name() override { return "SSH Store"; }
std::string doc() override;
};
struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store
{
// Hack for getting remote build log output.
// Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
// the documentation
const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"};
struct Connection;
std::string host;
ref<Pool<Connection>> connections;
SSHMaster master;
static std::set<std::string> uriSchemes() { return {"ssh"}; }
LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params);
ref<Connection> openConnection();
std::string getUri() override;
void queryPathInfoUncached(const StorePath & path,
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs) override;
void narFromPath(const StorePath & path, Sink & sink) override;
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
{ unsupported("queryPathFromHashPart"); }
StorePath addToStore(
std::string_view name,
const Path & srcPath,
FileIngestionMethod method,
HashAlgorithm hashAlgo,
PathFilter & filter,
RepairFlag repair,
const StorePathSet & references) override
{ unsupported("addToStore"); }
StorePath addTextToStore(
std::string_view name,
std::string_view s,
const StorePathSet & references,
RepairFlag repair) override
{ unsupported("addTextToStore"); }
private:
void putBuildSettings(Connection & conn);
public:
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override;
void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override;
void ensurePath(const StorePath & path) override
{ unsupported("ensurePath"); }
virtual ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
{ unsupported("getFSAccessor"); }
/**
* The default instance would schedule the work on the client side, but
* for consistency with `buildPaths` and `buildDerivation` it should happen
* on the remote side.
*
* We make this fail for now so we can add implement this properly later
* without it being a breaking change.
*/
void repairPath(const StorePath & path) override
{ unsupported("repairPath"); }
void computeFSClosure(const StorePathSet & paths,
StorePathSet & out, bool flipDirection = false,
bool includeOutputs = false, bool includeDerivers = false) override;
StorePathSet queryValidPaths(const StorePathSet & paths,
SubstituteFlag maybeSubstitute = NoSubstitute) override;
void connect() override;
unsigned int getProtocol() override;
/**
* The legacy ssh protocol doesn't support checking for trusted-user.
* Try using ssh-ng:// instead if you want to know.
*/
std::optional<TrustedFlag> isTrustedClient() override
{
return std::nullopt;
}
void queryRealisationUncached(const DrvOutput &,
Callback<std::shared_ptr<const Realisation>> callback) noexcept override
// TODO: Implement
{ unsupported("queryRealisation"); }
};
}