diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index bec668001..5a4ecf035 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -4,6 +4,7 @@ #include "nix/util/config-global.hh" #include "nix/util/serialise.hh" #include "nix/expr/eval-gc.hh" +#include "nix/expr/value.hh" #include "expr-config-private.hh" @@ -52,6 +53,13 @@ static inline void initGCReal() GC_INIT(); + /* Register valid displacements in case we are using alignment niches + for storing the type information. This way tagged pointers are considered + to be valid, even when they are not aligned. */ + if constexpr (detail::useBitPackedValueStorage) + for (std::size_t i = 1; i < sizeof(std::uintptr_t); ++i) + GC_register_displacement(i); + GC_set_oom_fn(oomHandler); /* Set the initial heap size to something fairly big (25% of diff --git a/src/libexpr/include/nix/expr/value.hh b/src/libexpr/include/nix/expr/value.hh index 3f28aca01..098effa29 100644 --- a/src/libexpr/include/nix/expr/value.hh +++ b/src/libexpr/include/nix/expr/value.hh @@ -19,23 +19,35 @@ namespace nix { struct Value; class BindingsBuilder; +/** + * Internal type discriminator, which is more detailed than `ValueType`, as + * it specifies the exact representation used (for types that have multiple + * possible representations). + * + * @warning The ordering is very significant. See ValueStorage::getInternalType() for details + * about how this is mapped into the alignment bits to save significant memory. + * This also restricts the number of internal types represented with distinct memory layouts. + */ typedef enum { tUninitialized = 0, + /* layout: Single/zero field payload */ tInt = 1, tBool, + tNull, + tFloat, + tExternal, + tPrimOp, + tAttrs, + /* layout: Pair of pointers payload */ + tListSmall, + tPrimOpApp, + tApp, + tThunk, + tLambda, + /* layout: Single untaggable field */ + tListN, tString, tPath, - tNull, - tAttrs, - tListSmall, - tListN, - tThunk, - tApp, - tLambda, - tPrimOp, - tPrimOpApp, - tExternal, - tFloat } InternalType; /** @@ -307,7 +319,7 @@ inline constexpr InternalType payloadTypeToInternalType = PayloadTypeToInternalT * All specializations of this type need to implement getStorage, setStorage and * getInternalType methods. */ -template +template class ValueStorage : public detail::ValueBase { protected: @@ -351,6 +363,287 @@ protected: } }; +namespace detail { + +/* Whether to use a specialization of ValueStorage that does bitpacking into + alignment niches. */ +template +inline constexpr bool useBitPackedValueStorage = (ptrSize == 8) && (__STDCPP_DEFAULT_NEW_ALIGNMENT__ >= 8); + +} // namespace detail + +/** + * Value storage that is optimized for 64 bit systems. + * Packs discriminator bits into the pointer alignment niches. + */ +template +class ValueStorage>> : public detail::ValueBase +{ + /* Needs a dependent type name in order for member functions (and + * potentially ill-formed bit casts) to be SFINAE'd out. + * + * Otherwise some member functions could possibly be instantiated for 32 bit + * systems and fail due to an unsatisfied constraint. + */ + template + struct PackedPointerTypeStruct + { + using type = std::uint64_t; + }; + + using PackedPointer = typename PackedPointerTypeStruct::type; + using Payload = std::array; + Payload payload = {}; + + static constexpr int discriminatorBits = 3; + static constexpr PackedPointer discriminatorMask = (PackedPointer(1) << discriminatorBits) - 1; + + /** + * The value is stored as a pair of 8-byte double words. All pointers are assumed + * to be 8-byte aligned. This gives us at most 6 bits of discriminator bits + * of free storage. In some cases when one double word can't be tagged the whole + * discriminator is stored in the first double word. + * + * The layout of discriminator bits is determined by the 3 bits of PrimaryDiscriminator, + * which are always stored in the lower 3 bits of the first dword of the payload. + * The memory layout has 3 types depending on the PrimaryDiscriminator value. + * + * PrimaryDiscriminator::pdSingleDWord - Only the second dword carries the data. + * That leaves the first 8 bytes free for storing the InternalType in the upper + * bits. + * + * PrimaryDiscriminator::pdListN - pdPath - Only has 3 available padding bits + * because: + * - tListN needs a size, whose lower bits we can't borrow. + * - tString and tPath have C-string fields, which don't necessarily need to + * be aligned. + * + * In this case we reserve their discriminators directly in the PrimaryDiscriminator + * bits stored in payload[0]. + * + * PrimaryDiscriminator::pdPairOfPointers - Payloads that consist of a pair of pointers. + * In this case the 3 lower bits of payload[1] can be tagged. + * + * The primary discriminator with value 0 is reserved for uninitialized Values, + * which are useful for diagnostics in C bindings. + */ + enum PrimaryDiscriminator : int { + pdUninitialized = 0, + pdSingleDWord, //< layout: Single/zero field payload + /* The order of these enumations must be the same as in InternalType. */ + pdListN, //< layout: Single untaggable field. + pdString, + pdPath, + pdPairOfPointers, //< layout: Pair of pointers payload + }; + + template + requires std::is_pointer_v + static T untagPointer(PackedPointer val) noexcept + { + return std::bit_cast(val & ~discriminatorMask); + } + + PrimaryDiscriminator getPrimaryDiscriminator() const noexcept + { + return static_cast(payload[0] & discriminatorMask); + } + + static void assertAligned(PackedPointer val) noexcept + { + assert((val & discriminatorMask) == 0 && "Pointer is not 8 bytes aligned"); + } + + template + void setSingleDWordPayload(PackedPointer untaggedVal) noexcept + { + /* There's plenty of free upper bits in the first dword, which is + used only for the discriminator. */ + payload[0] = static_cast(pdSingleDWord) | (static_cast(type) << discriminatorBits); + payload[1] = untaggedVal; + } + + template + void setUntaggablePayload(T * firstPtrField, U untaggableField) noexcept + { + static_assert(discriminator >= pdListN && discriminator <= pdPath); + auto firstFieldPayload = std::bit_cast(firstPtrField); + assertAligned(firstFieldPayload); + payload[0] = static_cast(discriminator) | firstFieldPayload; + payload[1] = std::bit_cast(untaggableField); + } + + template + void setPairOfPointersPayload(T * firstPtrField, U * secondPtrField) noexcept + { + static_assert(type >= tListSmall && type <= tLambda); + { + auto firstFieldPayload = std::bit_cast(firstPtrField); + assertAligned(firstFieldPayload); + payload[0] = static_cast(pdPairOfPointers) | firstFieldPayload; + } + { + auto secondFieldPayload = std::bit_cast(secondPtrField); + assertAligned(secondFieldPayload); + payload[1] = (type - tListSmall) | secondFieldPayload; + } + } + + template + requires std::is_pointer_v && std::is_pointer_v + void getPairOfPointersPayload(T & firstPtrField, U & secondPtrField) const noexcept + { + firstPtrField = untagPointer(payload[0]); + secondPtrField = untagPointer(payload[1]); + } + +protected: + /** Get internal type currently occupying the storage. */ + InternalType getInternalType() const noexcept + { + switch (auto pd = getPrimaryDiscriminator()) { + case pdUninitialized: + /* Discriminator value of zero is used to distinguish uninitialized values. */ + return tUninitialized; + case pdSingleDWord: + /* Payloads that only use up a single double word store the InternalType + in the upper bits of the first double word. */ + return InternalType(payload[0] >> discriminatorBits); + /* The order must match that of the enumerations defined in InternalType. */ + case pdListN: + case pdString: + case pdPath: + return static_cast(tListN + (pd - pdListN)); + case pdPairOfPointers: + return static_cast(tListSmall + (payload[1] & discriminatorMask)); + [[unlikely]] default: + unreachable(); + } + } + +#define NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(TYPE, MEMBER_A, MEMBER_B) \ + \ + void getStorage(TYPE & val) const noexcept \ + { \ + getPairOfPointersPayload(val MEMBER_A, val MEMBER_B); \ + } \ + \ + void setStorage(TYPE val) noexcept \ + { \ + setPairOfPointersPayload>(val MEMBER_A, val MEMBER_B); \ + } + + NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(SmallList, [0], [1]) + NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(PrimOpApplicationThunk, .left, .right) + NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(FunctionApplicationThunk, .left, .right) + NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(ClosureThunk, .env, .expr) + NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(Lambda, .env, .fun) + +#undef NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS + + void getStorage(NixInt & integer) const noexcept + { + /* PackedPointerType -> int64_t here is well-formed, since the standard requires + this conversion to follow 2's complement rules. This is just a no-op. */ + integer = NixInt(payload[1]); + } + + void getStorage(bool & boolean) const noexcept + { + boolean = payload[1]; + } + + void getStorage(Null & null) const noexcept {} + + void getStorage(NixFloat & fpoint) const noexcept + { + fpoint = std::bit_cast(payload[1]); + } + + void getStorage(ExternalValueBase *& external) const noexcept + { + external = std::bit_cast(payload[1]); + } + + void getStorage(PrimOp *& primOp) const noexcept + { + primOp = std::bit_cast(payload[1]); + } + + void getStorage(Bindings *& attrs) const noexcept + { + attrs = std::bit_cast(payload[1]); + } + + void getStorage(List & list) const noexcept + { + list.elems = untagPointer(payload[0]); + list.size = payload[1]; + } + + void getStorage(StringWithContext & string) const noexcept + { + string.context = untagPointer(payload[0]); + string.c_str = std::bit_cast(payload[1]); + } + + void getStorage(Path & path) const noexcept + { + path.accessor = untagPointer(payload[0]); + path.path = std::bit_cast(payload[1]); + } + + void setStorage(NixInt integer) noexcept + { + setSingleDWordPayload(integer.value); + } + + void setStorage(bool boolean) noexcept + { + setSingleDWordPayload(boolean); + } + + void setStorage(Null path) noexcept + { + setSingleDWordPayload(0); + } + + void setStorage(NixFloat fpoint) noexcept + { + setSingleDWordPayload(std::bit_cast(fpoint)); + } + + void setStorage(ExternalValueBase * external) noexcept + { + setSingleDWordPayload(std::bit_cast(external)); + } + + void setStorage(PrimOp * primOp) noexcept + { + setSingleDWordPayload(std::bit_cast(primOp)); + } + + void setStorage(Bindings * bindings) noexcept + { + setSingleDWordPayload(std::bit_cast(bindings)); + } + + void setStorage(List list) noexcept + { + setUntaggablePayload(list.elems, list.size); + } + + void setStorage(StringWithContext string) noexcept + { + setUntaggablePayload(string.context, string.c_str); + } + + void setStorage(Path path) noexcept + { + setUntaggablePayload(path.accessor, path.path); + } +}; + /** * View into a list of Value * that is itself immutable. *