mirror of
https://github.com/NixOS/nix
synced 2025-06-25 10:41:16 +02:00
store Symbols in a table as well, like positions
this slightly increases the amount of memory used for any given symbol, but this increase is more than made up for if the symbol is referenced more than once in the EvalState that holds it. on average every symbol should be referenced at least twice (once to introduce a binding, once to use it), so we expect no increase in memory on average. symbol tables are limited to 2³² entries like position tables, and similar arguments apply to why overflow is not likely: 2³² symbols would require as many string instances (at 24 bytes each) and map entries (at 24 bytes or more each, assuming that the map holds on average at most one item per bucket as the docs say). a full symbol table would require at least 192GB of memory just for symbols, which is well out of reach. (an ofborg eval of nixpks today creates less than a million symbols!)
This commit is contained in:
pennae
parent
00a3280232
commit
8775be3393
32 changed files with 525 additions and 448 deletions
|
|
@ -1,5 +1,7 @@
|
|||
#include "nixexpr.hh"
|
||||
#include "derivations.hh"
|
||||
#include "eval.hh"
|
||||
#include "symbol-table.hh"
|
||||
#include "util.hh"
|
||||
|
||||
#include <cstdlib>
|
||||
|
|
@ -10,12 +12,6 @@ namespace nix {
|
|||
|
||||
/* Displaying abstract syntax trees. */
|
||||
|
||||
std::ostream & operator << (std::ostream & str, const Expr & e)
|
||||
{
|
||||
e.show(str);
|
||||
return str;
|
||||
}
|
||||
|
||||
static void showString(std::ostream & str, std::string_view s)
|
||||
{
|
||||
str << '"';
|
||||
|
|
@ -54,81 +50,101 @@ static void showId(std::ostream & str, std::string_view s)
|
|||
|
||||
std::ostream & operator << (std::ostream & str, const Symbol & sym)
|
||||
{
|
||||
showId(str, *sym.s);
|
||||
showId(str, sym.s);
|
||||
return str;
|
||||
}
|
||||
|
||||
void Expr::show(std::ostream & str) const
|
||||
void Expr::show(const SymbolTable & symbols, std::ostream & str) const
|
||||
{
|
||||
abort();
|
||||
}
|
||||
|
||||
void ExprInt::show(std::ostream & str) const
|
||||
void ExprInt::show(const SymbolTable & symbols, std::ostream & str) const
|
||||
{
|
||||
str << n;
|
||||
}
|
||||
|
||||
void ExprFloat::show(std::ostream & str) const
|
||||
void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const
|
||||
{
|
||||
str << nf;
|
||||
}
|
||||
|
||||
void ExprString::show(std::ostream & str) const
|
||||
void ExprString::show(const SymbolTable & symbols, std::ostream & str) const
|
||||
{
|
||||
showString(str, s);
|
||||
}
|
||||
|
||||
void ExprPath::show(std::ostream & str) const
|
||||
void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const
|
||||
{
|
||||
str << s;
|
||||
}
|
||||
|
||||
void ExprVar::show(std::ostream & str) const
|
||||
void ExprVar::show(const SymbolTable & symbols, std::ostream & str) const
|
||||
{
|
||||
str << name;
|
||||
str << symbols[name];
|
||||
}
|
||||
|
||||
void ExprSelect::show(std::ostream & str) const
|
||||
void ExprSelect::show(const SymbolTable & symbols, std::ostream & str) const
|
||||
{
|
||||
str << "(" << *e << ")." << showAttrPath(attrPath);
|
||||
if (def) str << " or (" << *def << ")";
|
||||
str << "(";
|
||||
e->show(symbols, str);
|
||||
str << ")." << showAttrPath(symbols, attrPath);
|
||||
if (def) {
|
||||
str << " or (";
|
||||
def->show(symbols, str);
|
||||
str << ")";
|
||||
}
|
||||
}
|
||||
|
||||
void ExprOpHasAttr::show(std::ostream & str) const
|
||||
void ExprOpHasAttr::show(const SymbolTable & symbols, std::ostream & str) const
|
||||
{
|
||||
str << "((" << *e << ") ? " << showAttrPath(attrPath) << ")";
|
||||
str << "((";
|
||||
e->show(symbols, str);
|
||||
str << ") ? " << showAttrPath(symbols, attrPath) << ")";
|
||||
}
|
||||
|
||||
void ExprAttrs::show(std::ostream & str) const
|
||||
void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const
|
||||
{
|
||||
if (recursive) str << "rec ";
|
||||
str << "{ ";
|
||||
typedef const decltype(attrs)::value_type * Attr;
|
||||
std::vector<Attr> sorted;
|
||||
for (auto & i : attrs) sorted.push_back(&i);
|
||||
std::sort(sorted.begin(), sorted.end(), [](Attr a, Attr b) {
|
||||
return (const std::string &) a->first < (const std::string &) b->first;
|
||||
});
|
||||
std::sort(sorted.begin(), sorted.end(), [&](Attr a, Attr b) {
|
||||
std::string_view sa = symbols[a->first], sb = symbols[b->first];
|
||||
return sa < sb;
|
||||
});
|
||||
for (auto & i : sorted) {
|
||||
if (i->second.inherited)
|
||||
str << "inherit " << i->first << " " << "; ";
|
||||
else
|
||||
str << i->first << " = " << *i->second.e << "; ";
|
||||
str << "inherit " << symbols[i->first] << " " << "; ";
|
||||
else {
|
||||
str << symbols[i->first] << " = ";
|
||||
i->second.e->show(symbols, str);
|
||||
str << "; ";
|
||||
}
|
||||
}
|
||||
for (auto & i : dynamicAttrs) {
|
||||
str << "\"${";
|
||||
i.nameExpr->show(symbols, str);
|
||||
str << "}\" = ";
|
||||
i.valueExpr->show(symbols, str);
|
||||
str << "; ";
|
||||
}
|
||||
for (auto & i : dynamicAttrs)
|
||||
str << "\"${" << *i.nameExpr << "}\" = " << *i.valueExpr << "; ";
|
||||
str << "}";
|
||||
}
|
||||
|
||||
void ExprList::show(std::ostream & str) const
|
||||
void ExprList::show(const SymbolTable & symbols, std::ostream & str) const
|
||||
{
|
||||
str << "[ ";
|
||||
for (auto & i : elems)
|
||||
str << "(" << *i << ") ";
|
||||
for (auto & i : elems) {
|
||||
str << "(";
|
||||
i->show(symbols, str);
|
||||
str << ") ";
|
||||
}
|
||||
str << "]";
|
||||
}
|
||||
|
||||
void ExprLambda::show(std::ostream & str) const
|
||||
void ExprLambda::show(const SymbolTable & symbols, std::ostream & str) const
|
||||
{
|
||||
str << "(";
|
||||
if (hasFormals()) {
|
||||
|
|
@ -136,74 +152,100 @@ void ExprLambda::show(std::ostream & str) const
|
|||
bool first = true;
|
||||
for (auto & i : formals->formals) {
|
||||
if (first) first = false; else str << ", ";
|
||||
str << i.name;
|
||||
if (i.def) str << " ? " << *i.def;
|
||||
str << symbols[i.name];
|
||||
if (i.def) {
|
||||
str << " ? ";
|
||||
i.def->show(symbols, str);
|
||||
}
|
||||
}
|
||||
if (formals->ellipsis) {
|
||||
if (!first) str << ", ";
|
||||
str << "...";
|
||||
}
|
||||
str << " }";
|
||||
if (arg.set()) str << " @ ";
|
||||
if (arg) str << " @ ";
|
||||
}
|
||||
if (arg.set()) str << arg;
|
||||
str << ": " << *body << ")";
|
||||
if (arg) str << symbols[arg];
|
||||
str << ": ";
|
||||
body->show(symbols, str);
|
||||
str << ")";
|
||||
}
|
||||
|
||||
void ExprCall::show(std::ostream & str) const
|
||||
void ExprCall::show(const SymbolTable & symbols, std::ostream & str) const
|
||||
{
|
||||
str << '(' << *fun;
|
||||
str << '(';
|
||||
fun->show(symbols, str);
|
||||
for (auto e : args) {
|
||||
str << ' ';
|
||||
str << *e;
|
||||
e->show(symbols, str);
|
||||
}
|
||||
str << ')';
|
||||
}
|
||||
|
||||
void ExprLet::show(std::ostream & str) const
|
||||
void ExprLet::show(const SymbolTable & symbols, std::ostream & str) const
|
||||
{
|
||||
str << "(let ";
|
||||
for (auto & i : attrs->attrs)
|
||||
if (i.second.inherited) {
|
||||
str << "inherit " << i.first << "; ";
|
||||
str << "inherit " << symbols[i.first] << "; ";
|
||||
}
|
||||
else
|
||||
str << i.first << " = " << *i.second.e << "; ";
|
||||
str << "in " << *body << ")";
|
||||
else {
|
||||
str << symbols[i.first] << " = ";
|
||||
i.second.e->show(symbols, str);
|
||||
str << "; ";
|
||||
}
|
||||
str << "in ";
|
||||
body->show(symbols, str);
|
||||
str << ")";
|
||||
}
|
||||
|
||||
void ExprWith::show(std::ostream & str) const
|
||||
void ExprWith::show(const SymbolTable & symbols, std::ostream & str) const
|
||||
{
|
||||
str << "(with " << *attrs << "; " << *body << ")";
|
||||
str << "(with ";
|
||||
attrs->show(symbols, str);
|
||||
str << "; ";
|
||||
body->show(symbols, str);
|
||||
str << ")";
|
||||
}
|
||||
|
||||
void ExprIf::show(std::ostream & str) const
|
||||
void ExprIf::show(const SymbolTable & symbols, std::ostream & str) const
|
||||
{
|
||||
str << "(if " << *cond << " then " << *then << " else " << *else_ << ")";
|
||||
str << "(if ";
|
||||
cond->show(symbols, str);
|
||||
str << " then ";
|
||||
then->show(symbols, str);
|
||||
str << " else ";
|
||||
else_->show(symbols, str);
|
||||
str << ")";
|
||||
}
|
||||
|
||||
void ExprAssert::show(std::ostream & str) const
|
||||
void ExprAssert::show(const SymbolTable & symbols, std::ostream & str) const
|
||||
{
|
||||
str << "assert " << *cond << "; " << *body;
|
||||
str << "assert ";
|
||||
cond->show(symbols, str);
|
||||
str << "; ";
|
||||
body->show(symbols, str);
|
||||
}
|
||||
|
||||
void ExprOpNot::show(std::ostream & str) const
|
||||
void ExprOpNot::show(const SymbolTable & symbols, std::ostream & str) const
|
||||
{
|
||||
str << "(! " << *e << ")";
|
||||
str << "(! ";
|
||||
e->show(symbols, str);
|
||||
str << ")";
|
||||
}
|
||||
|
||||
void ExprConcatStrings::show(std::ostream & str) const
|
||||
void ExprConcatStrings::show(const SymbolTable & symbols, std::ostream & str) const
|
||||
{
|
||||
bool first = true;
|
||||
str << "(";
|
||||
for (auto & i : *es) {
|
||||
if (first) first = false; else str << " + ";
|
||||
str << *i.second;
|
||||
i.second->show(symbols, str);
|
||||
}
|
||||
str << ")";
|
||||
}
|
||||
|
||||
void ExprPos::show(std::ostream & str) const
|
||||
void ExprPos::show(const SymbolTable & symbols, std::ostream & str) const
|
||||
{
|
||||
str << "__curPos";
|
||||
}
|
||||
|
|
@ -234,16 +276,19 @@ std::ostream & operator << (std::ostream & str, const Pos & pos)
|
|||
}
|
||||
|
||||
|
||||
std::string showAttrPath(const AttrPath & attrPath)
|
||||
std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath)
|
||||
{
|
||||
std::ostringstream out;
|
||||
bool first = true;
|
||||
for (auto & i : attrPath) {
|
||||
if (!first) out << '.'; else first = false;
|
||||
if (i.symbol.set())
|
||||
out << i.symbol;
|
||||
else
|
||||
out << "\"${" << *i.expr << "}\"";
|
||||
if (i.symbol)
|
||||
out << symbols[i.symbol];
|
||||
else {
|
||||
out << "\"${";
|
||||
i.expr->show(symbols, out);
|
||||
out << "}\"";
|
||||
}
|
||||
}
|
||||
return out.str();
|
||||
}
|
||||
|
|
@ -252,28 +297,28 @@ std::string showAttrPath(const AttrPath & attrPath)
|
|||
|
||||
/* Computing levels/displacements for variables. */
|
||||
|
||||
void Expr::bindVars(const PosTable & pt, const StaticEnv & env)
|
||||
void Expr::bindVars(const EvalState & es, const StaticEnv & env)
|
||||
{
|
||||
abort();
|
||||
}
|
||||
|
||||
void ExprInt::bindVars(const PosTable & pt, const StaticEnv & env)
|
||||
void ExprInt::bindVars(const EvalState & es, const StaticEnv & env)
|
||||
{
|
||||
}
|
||||
|
||||
void ExprFloat::bindVars(const PosTable & pt, const StaticEnv & env)
|
||||
void ExprFloat::bindVars(const EvalState & es, const StaticEnv & env)
|
||||
{
|
||||
}
|
||||
|
||||
void ExprString::bindVars(const PosTable & pt, const StaticEnv & env)
|
||||
void ExprString::bindVars(const EvalState & es, const StaticEnv & env)
|
||||
{
|
||||
}
|
||||
|
||||
void ExprPath::bindVars(const PosTable & pt, const StaticEnv & env)
|
||||
void ExprPath::bindVars(const EvalState & es, const StaticEnv & env)
|
||||
{
|
||||
}
|
||||
|
||||
void ExprVar::bindVars(const PosTable & pt, const StaticEnv & env)
|
||||
void ExprVar::bindVars(const EvalState & es, const StaticEnv & env)
|
||||
{
|
||||
/* Check whether the variable appears in the environment. If so,
|
||||
set its level and displacement. */
|
||||
|
|
@ -299,31 +344,31 @@ void ExprVar::bindVars(const PosTable & pt, const StaticEnv & env)
|
|||
"undefined variable" error now. */
|
||||
if (withLevel == -1)
|
||||
throw UndefinedVarError({
|
||||
.msg = hintfmt("undefined variable '%1%'", name),
|
||||
.errPos = pt[pos]
|
||||
.msg = hintfmt("undefined variable '%1%'", es.symbols[name]),
|
||||
.errPos = es.positions[pos]
|
||||
});
|
||||
fromWith = true;
|
||||
this->level = withLevel;
|
||||
}
|
||||
|
||||
void ExprSelect::bindVars(const PosTable & pt, const StaticEnv & env)
|
||||
void ExprSelect::bindVars(const EvalState & es, const StaticEnv & env)
|
||||
{
|
||||
e->bindVars(pt, env);
|
||||
if (def) def->bindVars(pt, env);
|
||||
e->bindVars(es, env);
|
||||
if (def) def->bindVars(es, env);
|
||||
for (auto & i : attrPath)
|
||||
if (!i.symbol.set())
|
||||
i.expr->bindVars(pt, env);
|
||||
if (!i.symbol)
|
||||
i.expr->bindVars(es, env);
|
||||
}
|
||||
|
||||
void ExprOpHasAttr::bindVars(const PosTable & pt, const StaticEnv & env)
|
||||
void ExprOpHasAttr::bindVars(const EvalState & es, const StaticEnv & env)
|
||||
{
|
||||
e->bindVars(pt, env);
|
||||
e->bindVars(es, env);
|
||||
for (auto & i : attrPath)
|
||||
if (!i.symbol.set())
|
||||
i.expr->bindVars(pt, env);
|
||||
if (!i.symbol)
|
||||
i.expr->bindVars(es, env);
|
||||
}
|
||||
|
||||
void ExprAttrs::bindVars(const PosTable & pt, const StaticEnv & env)
|
||||
void ExprAttrs::bindVars(const EvalState & es, const StaticEnv & env)
|
||||
{
|
||||
const StaticEnv * dynamicEnv = &env;
|
||||
StaticEnv newEnv(false, &env, recursive ? attrs.size() : 0);
|
||||
|
|
@ -338,35 +383,35 @@ void ExprAttrs::bindVars(const PosTable & pt, const StaticEnv & env)
|
|||
// No need to sort newEnv since attrs is in sorted order.
|
||||
|
||||
for (auto & i : attrs)
|
||||
i.second.e->bindVars(pt, i.second.inherited ? env : newEnv);
|
||||
i.second.e->bindVars(es, i.second.inherited ? env : newEnv);
|
||||
}
|
||||
|
||||
else
|
||||
for (auto & i : attrs)
|
||||
i.second.e->bindVars(pt, env);
|
||||
i.second.e->bindVars(es, env);
|
||||
|
||||
for (auto & i : dynamicAttrs) {
|
||||
i.nameExpr->bindVars(pt, *dynamicEnv);
|
||||
i.valueExpr->bindVars(pt, *dynamicEnv);
|
||||
i.nameExpr->bindVars(es, *dynamicEnv);
|
||||
i.valueExpr->bindVars(es, *dynamicEnv);
|
||||
}
|
||||
}
|
||||
|
||||
void ExprList::bindVars(const PosTable & pt, const StaticEnv & env)
|
||||
void ExprList::bindVars(const EvalState & es, const StaticEnv & env)
|
||||
{
|
||||
for (auto & i : elems)
|
||||
i->bindVars(pt, env);
|
||||
i->bindVars(es, env);
|
||||
}
|
||||
|
||||
void ExprLambda::bindVars(const PosTable & pt, const StaticEnv & env)
|
||||
void ExprLambda::bindVars(const EvalState & es, const StaticEnv & env)
|
||||
{
|
||||
StaticEnv newEnv(
|
||||
false, &env,
|
||||
(hasFormals() ? formals->formals.size() : 0) +
|
||||
(!arg.set() ? 0 : 1));
|
||||
(!arg ? 0 : 1));
|
||||
|
||||
Displacement displ = 0;
|
||||
|
||||
if (arg.set()) newEnv.vars.emplace_back(arg, displ++);
|
||||
if (arg) newEnv.vars.emplace_back(arg, displ++);
|
||||
|
||||
if (hasFormals()) {
|
||||
for (auto & i : formals->formals)
|
||||
|
|
@ -375,20 +420,20 @@ void ExprLambda::bindVars(const PosTable & pt, const StaticEnv & env)
|
|||
newEnv.sort();
|
||||
|
||||
for (auto & i : formals->formals)
|
||||
if (i.def) i.def->bindVars(pt, newEnv);
|
||||
if (i.def) i.def->bindVars(es, newEnv);
|
||||
}
|
||||
|
||||
body->bindVars(pt, newEnv);
|
||||
body->bindVars(es, newEnv);
|
||||
}
|
||||
|
||||
void ExprCall::bindVars(const PosTable & pt, const StaticEnv & env)
|
||||
void ExprCall::bindVars(const EvalState & es, const StaticEnv & env)
|
||||
{
|
||||
fun->bindVars(pt, env);
|
||||
fun->bindVars(es, env);
|
||||
for (auto e : args)
|
||||
e->bindVars(pt, env);
|
||||
e->bindVars(es, env);
|
||||
}
|
||||
|
||||
void ExprLet::bindVars(const PosTable & pt, const StaticEnv & env)
|
||||
void ExprLet::bindVars(const EvalState & es, const StaticEnv & env)
|
||||
{
|
||||
StaticEnv newEnv(false, &env, attrs->attrs.size());
|
||||
|
||||
|
|
@ -399,12 +444,12 @@ void ExprLet::bindVars(const PosTable & pt, const StaticEnv & env)
|
|||
// No need to sort newEnv since attrs->attrs is in sorted order.
|
||||
|
||||
for (auto & i : attrs->attrs)
|
||||
i.second.e->bindVars(pt, i.second.inherited ? env : newEnv);
|
||||
i.second.e->bindVars(es, i.second.inherited ? env : newEnv);
|
||||
|
||||
body->bindVars(pt, newEnv);
|
||||
body->bindVars(es, newEnv);
|
||||
}
|
||||
|
||||
void ExprWith::bindVars(const PosTable & pt, const StaticEnv & env)
|
||||
void ExprWith::bindVars(const EvalState & es, const StaticEnv & env)
|
||||
{
|
||||
/* Does this `with' have an enclosing `with'? If so, record its
|
||||
level so that `lookupVar' can look up variables in the previous
|
||||
|
|
@ -418,57 +463,60 @@ void ExprWith::bindVars(const PosTable & pt, const StaticEnv & env)
|
|||
break;
|
||||
}
|
||||
|
||||
attrs->bindVars(pt, env);
|
||||
attrs->bindVars(es, env);
|
||||
StaticEnv newEnv(true, &env);
|
||||
body->bindVars(pt, newEnv);
|
||||
body->bindVars(es, newEnv);
|
||||
}
|
||||
|
||||
void ExprIf::bindVars(const PosTable & pt, const StaticEnv & env)
|
||||
void ExprIf::bindVars(const EvalState & es, const StaticEnv & env)
|
||||
{
|
||||
cond->bindVars(pt, env);
|
||||
then->bindVars(pt, env);
|
||||
else_->bindVars(pt, env);
|
||||
cond->bindVars(es, env);
|
||||
then->bindVars(es, env);
|
||||
else_->bindVars(es, env);
|
||||
}
|
||||
|
||||
void ExprAssert::bindVars(const PosTable & pt, const StaticEnv & env)
|
||||
void ExprAssert::bindVars(const EvalState & es, const StaticEnv & env)
|
||||
{
|
||||
cond->bindVars(pt, env);
|
||||
body->bindVars(pt, env);
|
||||
cond->bindVars(es, env);
|
||||
body->bindVars(es, env);
|
||||
}
|
||||
|
||||
void ExprOpNot::bindVars(const PosTable & pt, const StaticEnv & env)
|
||||
void ExprOpNot::bindVars(const EvalState & es, const StaticEnv & env)
|
||||
{
|
||||
e->bindVars(pt, env);
|
||||
e->bindVars(es, env);
|
||||
}
|
||||
|
||||
void ExprConcatStrings::bindVars(const PosTable & pt, const StaticEnv & env)
|
||||
void ExprConcatStrings::bindVars(const EvalState & es, const StaticEnv & env)
|
||||
{
|
||||
for (auto & i : *es)
|
||||
i.second->bindVars(pt, env);
|
||||
for (auto & i : *this->es)
|
||||
i.second->bindVars(es, env);
|
||||
}
|
||||
|
||||
void ExprPos::bindVars(const PosTable & pt, const StaticEnv & env)
|
||||
void ExprPos::bindVars(const EvalState & es, const StaticEnv & env)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
/* Storing function names. */
|
||||
|
||||
void Expr::setName(Symbol & name)
|
||||
void Expr::setName(SymbolIdx name)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
void ExprLambda::setName(Symbol & name)
|
||||
void ExprLambda::setName(SymbolIdx name)
|
||||
{
|
||||
this->name = name;
|
||||
body->setName(name);
|
||||
}
|
||||
|
||||
|
||||
std::string ExprLambda::showNamePos(const PosTable & pt) const
|
||||
std::string ExprLambda::showNamePos(const EvalState & state) const
|
||||
{
|
||||
return fmt("%1% at %2%", name.set() ? "'" + (std::string) name + "'" : "anonymous function", pt[pos]);
|
||||
std::string id(name
|
||||
? concatStrings("'", state.symbols[name], "'")
|
||||
: "anonymous function");
|
||||
return fmt("%1% at %2%", id, state.positions[pos]);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -478,8 +526,7 @@ std::string ExprLambda::showNamePos(const PosTable & pt) const
|
|||
size_t SymbolTable::totalSize() const
|
||||
{
|
||||
size_t n = 0;
|
||||
for (auto & i : store)
|
||||
n += i.size();
|
||||
dump([&] (const Symbol & s) { n += std::string_view(s).size(); });
|
||||
return n;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue