diff --git a/.gitmodules b/.gitmodules index 761a6f1..04bf1e6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "test-vectors/jamtestvectors"] path = test-vectors/jamtestvectors - url = https://github.com/w3f/jamtestvectors/ + url = https://github.com/davxy/jam-test-vectors diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ecad11..484c4c5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,12 @@ find_package(Boost CONFIG REQUIRED) find_package(fmt CONFIG REQUIRED) find_package(scale CONFIG REQUIRED) +add_library(headers INTERFACE) +target_include_directories(headers INTERFACE + $ + $ +) + add_subdirectory(src) if(TESTING) diff --git a/python/asn1.py b/python/asn1.py index 6a70a7d..5530dd2 100755 --- a/python/asn1.py +++ b/python/asn1.py @@ -2,26 +2,24 @@ import asn1tools import os.path +import sys + DIR = os.path.dirname(__file__) TEST_VECTORS_DIR = os.path.join(DIR, "../test-vectors") OUTPUT_NAME = "types" -OUTPUT_DIR = os.path.join(TEST_VECTORS_DIR, "safrole") -SAFROLE_DIR = os.path.join(TEST_VECTORS_DIR, "jamtestvectors/safrole") -ARGS = ["validators-count", "epoch-length"] -NS = "jam::test_vectors_safrole" def flatten(aaa: list[list]): return [a for aa in aaa for a in aa] -def safrole_file(name: str): - return os.path.join(SAFROLE_DIR, name + ".asn") +def asn_file(name: str): + return os.path.join(TEST_VECTORS_DIR, "jamtestvectors", name + ".asn") -def write(ext: str, lines: list[str]): - with open(os.path.join(OUTPUT_DIR, OUTPUT_NAME + ext), "w") as f: +def write(path: str, lines: list[str]): + with open(path, "w") as f: f.writelines(line + "\n" for line in lines) @@ -33,7 +31,7 @@ def c_struct(name: str, members): return [ "struct %s {" % name, *((" %s %s;" % (t, k) for k, t in members)), - " bool operator==(const %s &)const = default;" % name, + " bool operator==(const %s &) const = default;" % name, "};", ] @@ -49,19 +47,24 @@ def indent(lines): class Type: + ignore_args = True + def __init__(self, name: str): self.name = name self.args: list[str] = [] self.decl: list[str] = [] self.scale: list[str] = [] + self.diff: list[str] = [] def c_tdecl(self): - if not self.args: + if self.ignore_args or not self.args: return [] - return ["template<%s>" % ", ".join("uint32_t %s" % c_dash(x) for x in self.args)] + return [ + "template<%s>" % ", ".join("uint32_t %s" % c_dash(x) for x in self.args) + ] def c_targs(self): - if not self.args: + if self.ignore_args or not self.args: return "" return "<%s>" % ", ".join(c_dash(x) for x in self.args) @@ -98,7 +101,7 @@ def deep(k: str): return deps1, deps2 -def asn_args(types: dict, deps2: dict[str, set[str]]): +def asn_args(ARGS: list[str], types: dict, deps2: dict[str, set[str]]): args = { k: { s[0] @@ -122,10 +125,11 @@ def c_scale(ty: Type, encode: list[str], decode: list[str]): " return s;", "}", *ty.c_tdecl(), - "inline scale::ScaleDecoderStream &operator>>(scale::ScaleDecoderStream &s, %s &v) {" + "scale::ScaleDecoderStream &operator>>(scale::ScaleDecoderStream &s, %s &) = delete;" + % ty.c_tname(), + "void decodeConfig(scale::ScaleDecoderStream &s, %s &v, const auto &config) {" % ty.c_tname(), *indent(decode), - " return s;", "}", ] @@ -134,13 +138,22 @@ def c_scale_struct(ty: Type, members: list[str]): return c_scale( ty, ["s << v.%s;" % x for x in members], - ["s >> v.%s;" % x for x in members], + ["decodeConfig(s, v.%s, config);" % x for x in members], ) -def parse_types(path: str): +def c_diff(NS: str, ty: Type, lines: list[str]): + return [ + *ty.c_tdecl(), + "DIFF_F(%s::%s) {" % (NS, ty.c_tname()), + *indent(lines), + "}", + ] + + +def parse_types(NS: str, ARGS: list[str], path: str, key: str): def asn_sequence_of(t): - (size,) = t["size"] + (size,) = t.get("size", (None,)) fixed = isinstance(size, (int, str)) T = t["element"]["type"] assert T in asn_types @@ -149,10 +162,14 @@ def asn_sequence_of(t): return "qtils::BytesN<%s>" % c_dash(size) return "qtils::Bytes" if fixed: + if isinstance(size, str): + return "jam::ConfigVec<%s, ConfigField::%s>" % (T, c_dash(size)) return "std::array<%s, %s>" % (T, c_dash(size)) return "std::vector<%s>" % T def asn_member(t): + if t["type"] == "OCTET STRING": + t = dict(type="SEQUENCE OF", element=dict(type="U8"), size=t["size"]) if t["type"] == "SEQUENCE OF": r = asn_sequence_of(t) elif t["type"] in asn_types: @@ -163,10 +180,12 @@ def asn_member(t): return "std::optional<%s>" % r return r - asn_types: dict = asn1tools.parse_files([path])["SafroleModule"]["types"] + asn_types: dict = asn1tools.parse_files([path])[key]["types"] + if "U8" not in asn_types: + asn_types["U8"] = dict(type="INTEGER") deps1, deps2 = asn_deps(asn_types) types = {tname: Type(tname) for tname in asn_types} - for tname, args in asn_args(asn_types, deps2).items(): + for tname, args in asn_args(ARGS, asn_types, deps2).items(): types[tname].args = args enum_trait = [] for tname, t in asn_types.items(): @@ -177,7 +196,15 @@ def asn_member(t): if tname == "U32": ty.decl = c_using(tname, "uint32_t") continue + if t["type"] == "NULL": + t = dict(type="SEQUENCE", members=[]) if t["type"] == "CHOICE": + if tname == "MmrPeak": + assert [x["name"] for x in t["members"]] == ["none", "some"] + ty.decl = c_using( + tname, "std::optional<%s>" % asn_member(t["members"][1]) + ) + continue ty.decl = c_struct( tname, [ @@ -188,7 +215,7 @@ def asn_member(t): ) ], ) - ty.scale.extend(c_scale_struct(ty, ["v"])) + ty.scale += c_scale_struct(ty, ["v"]) continue if t["type"] == "ENUMERATED": values = [x[1] for x in t["values"]] @@ -198,14 +225,13 @@ def asn_member(t): *(" %s," % c_dash(x[0]) for x in t["values"]), "};", ] - ns = "%s::generic" % NS enum_trait.append( "SCALE_DEFINE_ENUM_VALUE_LIST(%s, %s, %s)" % ( - ns, + NS, ty.name, ", ".join( - "%s::%s::%s" % (ns, ty.name, c_dash(x[0])) for x in t["values"] + "%s::%s::%s" % (NS, ty.name, c_dash(x[0])) for x in t["values"] ), ) ) @@ -214,8 +240,9 @@ def asn_member(t): ty.decl = c_struct( tname, [(c_dash(x["name"]), asn_member(x)) for x in t["members"]] ) - ty.scale.extend( - c_scale_struct(ty, [c_dash(x["name"]) for x in t["members"]]) + ty.scale += c_scale_struct(ty, [c_dash(x["name"]) for x in t["members"]]) + ty.diff = c_diff( + NS, ty, ["DIFF_M(%s);" % c_dash(x["name"]) for x in t["members"]] ) continue ty.decl = c_using(tname, asn_member(t)) @@ -224,49 +251,94 @@ def asn_member(t): return [types[k] for k in order], enum_trait -def parse_const(path: str): - values: dict = asn1tools.parse_files([path])["SafroleConstants"]["values"] +def parse_const(path: str, key: str): + values: dict = asn1tools.parse_files([path])[key]["values"] assert all(v["type"] == "INTEGER" for v in values.values()) return {k: v["value"] for k, v in values.items()} -types, enum_trait = parse_types(safrole_file("safrole")) +class Gen: + def __init__(self, NS: str, ARGS: list[str], path: str, key: str, configs): + g_config = [ + "struct ConfigField {", + *[" struct %s {};" % c_dash(a) for a in ARGS], + "};", + "struct Config {", + *[" uint32_t %s;" % c_dash(a) for a in ARGS], + *[ + " auto get(ConfigField::%s) const { return %s; }" + % (c_dash(a), c_dash(a)) + for a in ARGS + ], + "};", + ] + for name, args in configs: + g_config += [ + "constexpr Config config_%s {" % name, + *[" .%s = %s," % (c_dash(a), args[a]) for a in ARGS], + "};", + ] + self.types, self.enum_trait = parse_types(NS, ARGS, path, key) + self.g_types = flatten([*ty.c_tdecl(), *ty.decl] for ty in self.types) + self.g_types = g_config + self.g_types + self.g_types = ["namespace %s {" % NS, *indent(self.g_types), "}"] + self.g_types = [ + "#pragma once", + "#include ", + "#include ", + "#include ", + *self.g_types, + ] + self.g_scale = flatten(ty.scale for ty in self.types) + self.g_scale = ["namespace %s {" % NS, *indent(self.g_scale), "}"] + self.g_scale = [ + "#pragma once", + "#include ", + "#include ", + '#include "%s.hpp"' % OUTPUT_NAME, + *self.g_scale, + *self.enum_trait, + ] + self.g_diff = flatten(ty.diff for ty in self.types) + self.g_diff = [ + "#pragma once", + "#include ", + '#include "%s.hpp"' % OUTPUT_NAME, + *self.g_diff, + ] + + def write(self, name: str): + prefix = os.path.join(TEST_VECTORS_DIR, name, OUTPUT_NAME) + write(prefix + ".hpp", self.g_types) + write(prefix + ".scale.hpp", self.g_scale) + write(prefix + ".diff.hpp", self.g_diff) -g_types = flatten([*ty.c_tdecl(), *ty.decl] for ty in types) -g_types = ["namespace %s::generic {" % NS, *indent(g_types), "}"] -for name in ["tiny", "full"]: - args = parse_const(safrole_file(name)) - g_args = [ - *["static constexpr uint32_t %s = %s;" % (c_dash(a), args[a]) for a in ARGS], - *flatten( - c_using(ty.name, "%s::generic::%s" % (NS, ty.c_tname())) for ty in types - ), - ] - g_args = ["struct %s {" % name, *indent(g_args), "};"] - g_types.extend( +def safrole(): + g = Gen( + "jam::test_vectors_safrole", + ["validators-count", "epoch-length"], + asn_file("safrole/safrole"), + "SafroleModule", [ - "namespace %s {" % NS, - *indent(g_args), - "}", - ] + (name, parse_const(asn_file("safrole/%s" % name), "Constants")) + for name in ["tiny", "full"] + ], + ) + g.write("safrole") + + +def history(): + g = Gen( + "jam::test_vectors_history", + [], + asn_file("history/history"), + "HistoryModule", + [], ) -g_types = [ - "#pragma once", - "#include ", - "#include ", - *g_types, -] - -g_scale = flatten(ty.scale for ty in types) -g_scale = ["namespace %s::generic {" % NS, *indent(g_scale), "}"] -g_scale = [ - "#pragma once", - "#include ", - '#include "%s.hpp"' % OUTPUT_NAME, - *g_scale, - *enum_trait, -] - -write(".hpp", g_types) -write(".scale.hpp", g_scale) + g.write("history") + + +if __name__ == "__main__": + for arg in sys.argv[1:]: + dict(safrole=safrole, history=history)[arg]() diff --git a/src/jam/keccak.hpp b/src/jam/keccak.hpp new file mode 100644 index 0000000..d5e3e5c --- /dev/null +++ b/src/jam/keccak.hpp @@ -0,0 +1,113 @@ +// https://github.com/nayuki/Bitcoin-Cryptography-Library/blob/master/cpp/Keccak256.hpp +// Not using https://vcpkg.io/en/package/keccak-tiny from vcpkg because it only +// exports sha3, not keccak + +#pragma once + +#include + +/** + * Keccak hash + */ + +namespace jam { + struct Keccak { + using Hash32 = qtils::BytesN<32>; + uint64_t state[5][5] = {}; + size_t blockOff = 0; + static constexpr size_t HASH_LEN = 32; + static constexpr size_t BLOCK_SIZE = 200 - HASH_LEN * 2; + void absorb() { + auto rotl64 = [](uint64_t x, uint8_t i) { + return (x << i) | (x >> (64 - i)); + }; + constexpr size_t NUM_ROUNDS = 24; + constexpr uint8_t ROTATION[5][5] = { + {0, 36, 3, 41, 18}, + {1, 44, 10, 45, 2}, + {62, 6, 43, 15, 61}, + {28, 55, 25, 21, 56}, + {27, 20, 39, 8, 14}, + }; + uint64_t(*a)[5] = state; + uint8_t r = 1; // LFSR + for (int i = 0; i < NUM_ROUNDS; i++) { + // Theta step + uint64_t c[5] = {}; + for (int x = 0; x < 5; x++) { + for (int y = 0; y < 5; y++) { + c[x] ^= a[x][y]; + } + } + for (int x = 0; x < 5; x++) { + uint64_t d = c[(x + 4) % 5] ^ rotl64(c[(x + 1) % 5], 1); + for (int y = 0; y < 5; y++) { + a[x][y] ^= d; + } + } + + // Rho and pi steps + uint64_t b[5][5]; + for (int x = 0; x < 5; x++) { + for (int y = 0; y < 5; y++) { + b[y][(x * 2 + y * 3) % 5] = rotl64(a[x][y], ROTATION[x][y]); + } + } + + // Chi step + for (int x = 0; x < 5; x++) { + for (int y = 0; y < 5; y++) { + a[x][y] = b[x][y] ^ (~b[(x + 1) % 5][y] & b[(x + 2) % 5][y]); + } + } + + // Iota step + for (int j = 0; j < 7; j++) { + a[0][0] ^= static_cast(r & 1) << ((1 << j) - 1); + r = static_cast((r << 1) ^ ((r >> 7) * 0x171)); + } + } + } + Hash32 finalize() { + Hash32 hash; + // Final block and padding + { + int i = blockOff >> 3; + state[i % 5][i / 5] ^= UINT64_C(0x01) << ((blockOff & 7) << 3); + blockOff = BLOCK_SIZE - 1; + int j = blockOff >> 3; + state[j % 5][j / 5] ^= UINT64_C(0x80) << ((blockOff & 7) << 3); + absorb(); + } + // Uint64 array to bytes in little endian + for (int i = 0; i < HASH_LEN; i++) { + int j = i >> 3; + hash[i] = static_cast(state[j % 5][j / 5] >> ((i & 7) << 3)); + } + return hash; + } + void update(uint8_t byte) { + int j = blockOff >> 3; + state[j % 5][j / 5] ^= static_cast(byte) + << ((blockOff & 7) << 3); + blockOff++; + if (blockOff == BLOCK_SIZE) { + absorb(); + blockOff = 0; + } + } + Keccak &update(qtils::BytesIn input) { + for (auto &x : input) { + update(x); + } + return *this; + } + Hash32 hash() const { + auto copy = *this; + return copy.finalize(); + } + static Hash32 hash(qtils::BytesIn input) { + return Keccak{}.update(input).hash(); + } + }; +} // namespace jam diff --git a/test-vectors/CMakeLists.txt b/test-vectors/CMakeLists.txt index bf96af0..81ac577 100644 --- a/test-vectors/CMakeLists.txt +++ b/test-vectors/CMakeLists.txt @@ -4,4 +4,12 @@ # SPDX-License-Identifier: Apache-2.0 # +add_library(test_vectors_headers INTERFACE) +target_include_directories(test_vectors_headers INTERFACE + ${PROJECT_SOURCE_DIR} +) + +include(asn1.cmake) + +add_subdirectory(history) add_subdirectory(safrole) diff --git a/test-vectors/asn1.cmake b/test-vectors/asn1.cmake new file mode 100644 index 0000000..3d796ed --- /dev/null +++ b/test-vectors/asn1.cmake @@ -0,0 +1,36 @@ +# +# Copyright Quadrivium LLC +# All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# + +function(asn1 name) + set(ASN_DIR ${PROJECT_SOURCE_DIR}/test-vectors/jamtestvectors/${name}) + foreach(x IN LISTS ARGN) + list(APPEND ASN_FILES ${ASN_DIR}/${x}.asn) + endforeach() + set(OUTPUT_PREFIX ${PROJECT_SOURCE_DIR}/test-vectors/${name}/types) + set(OUTPUT ${OUTPUT_PREFIX}.hpp ${OUTPUT_PREFIX}.scale.hpp) + set(ASN1_PY ${PROJECT_SOURCE_DIR}/python/asn1.py) + add_custom_command( + OUTPUT ${OUTPUT} + COMMAND ${Python3_EXECUTABLE} ${ASN1_PY} ${name} + DEPENDS ${ASN1_PY} ${ASN_FILES} + ) + + add_library(test_vectors_${name}_types INTERFACE ${OUTPUT}) + target_link_libraries(test_vectors_${name}_types INTERFACE + scale::scale + test_vectors_headers + ) + + add_executable(test_vectors_${name}_types_test + types.test.cpp + ) + target_compile_definitions(test_vectors_${name}_types_test PRIVATE PROJECT_SOURCE_DIR="${PROJECT_SOURCE_DIR}") + target_link_libraries(test_vectors_${name}_types_test + fmt::fmt + test_vectors_${name}_types + ) + add_test(test_vectors_${name}_types_test test_vectors_${name}_types_test) +endfunction() diff --git a/test-vectors/common.hpp b/test-vectors/common.hpp new file mode 100644 index 0000000..2a7f982 --- /dev/null +++ b/test-vectors/common.hpp @@ -0,0 +1,32 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +/** + * Common functions used in tests + */ + +namespace jam { + // [GP 0.4.3 3.7.2] + // https://github.com/gavofyork/graypaper/blob/v0.4.3/text/notation.tex#L132 + template + qtils::BytesN frown( + const qtils::BytesN &x, const qtils::BytesN &y) { + qtils::BytesN xy; + memcpy(xy.data(), x.data(), X); + memcpy(xy.data() + X, y.data(), Y); + return xy; + } + + // [GP 0.4.3 3.8.1] + // https://github.com/gavofyork/graypaper/blob/v0.4.3/text/notation.tex#L153 + inline auto mathcal_H_K(qtils::BytesIn m) { + return Keccak::hash(m); + } +} // namespace jam diff --git a/test-vectors/config-types-scale.hpp b/test-vectors/config-types-scale.hpp new file mode 100644 index 0000000..4dc92d2 --- /dev/null +++ b/test-vectors/config-types-scale.hpp @@ -0,0 +1,94 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +namespace scale { + template + void decodeConfig(ScaleDecoderStream &s, T &v, const auto &config) { + s >> v; + } + + template + void decodeConfig( + ScaleDecoderStream &s, std::vector &v, const auto &config) { + auto n = s.decodeLength(); + v.resize(0); + v.reserve(n); + for (size_t i = 0; i < n; ++i) { + T item; + decodeConfig(s, item, config); + v.emplace_back(std::move(item)); + } + } + + template + void decodeConfig( + ScaleDecoderStream &s, std::optional &v, const auto &config) { + uint8_t i = 0; + s >> i; + if (i == 0) { + v.reset(); + } else if (i == 1) { + T item; + decodeConfig(s, item, config); + v = std::move(item); + } else { + raise(DecodeError::WRONG_TYPE_INDEX); + } + } + + template + void decodeConfigVariant( + size_t i, ScaleDecoderStream &s, auto &v, const auto &config) { + if (i == I) { + T item; + decodeConfig(s, item, config); + v = std::move(item); + } else if constexpr (sizeof...(Ts) != 0) { + decodeConfigVariant(i, s, v, config); + } + } + + template + void decodeConfig( + ScaleDecoderStream &s, boost::variant &v, const auto &config) { + uint8_t i = 0; + s >> i; + if (i >= sizeof...(Ts)) { + raise(DecodeError::WRONG_TYPE_INDEX); + } + decodeConfigVariant<0, Ts...>(i, s, v, config); + } +} // namespace scale + +namespace jam { + template + void decodeConfig(scale::ScaleDecoderStream &s, + ConfigVec &v, + const auto &config) { + auto n = config.get(ConfigField{}); + v.v.resize(0); + v.v.reserve(n); + for (size_t i = 0; i < n; ++i) { + T item; + decodeConfig(s, item, config); + v.v.emplace_back(std::move(item)); + } + } + + template + scale::ScaleEncoderStream &operator<<( + scale::ScaleEncoderStream &s, const ConfigVec &v) { + for (auto &item : v.v) { + s << item; + } + return s; + } +} // namespace jam diff --git a/test-vectors/config-types.hpp b/test-vectors/config-types.hpp new file mode 100644 index 0000000..1bcd962 --- /dev/null +++ b/test-vectors/config-types.hpp @@ -0,0 +1,18 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace jam { + template + struct ConfigVec { + std::vector v; + + bool operator==(const ConfigVec &) const = default; + }; +} // namespace jam diff --git a/test-vectors/diff.hpp b/test-vectors/diff.hpp new file mode 100644 index 0000000..12b20fe --- /dev/null +++ b/test-vectors/diff.hpp @@ -0,0 +1,117 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +/** + * Print colorful diff for objects. + * Used for debugging tests. + */ + +struct Indent { + int indent = 0; + Indent operator~() const { + return Indent{indent + 1}; + } +}; +template <> +struct fmt::formatter { + static constexpr auto parse(format_parse_context &ctx) { + return ctx.begin(); + } + auto format(const Indent &v, format_context &ctx) const { + auto out = ctx.out(); + for (int i = 0; i < v.indent; ++i) { + out = fmt::detail::write(out, " "); + } + return out; + } +}; + +#define DIFF_F(...) \ + void diff(Indent indent, const __VA_ARGS__ &v1, const __VA_ARGS__ &v2) +#define DIFF_M(m) diff_m(indent, v1.m, v2.m, #m) + +enum class color { red = 31, green = 32 }; +template +void _diff(color color, fmt::format_string f, T &&...a) { + fmt::print("\x1b[0;{}m", (int)color); + fmt::print(f, static_cast(a)...); + fmt::println("\x1b[0;0m"); +} +template +void diff1(fmt::format_string f, T &&...a) { + _diff(color::red, f, static_cast(a)...); +} +template +void diff2(fmt::format_string f, T &&...a) { + _diff(color::green, f, static_cast(a)...); +} + +template +void diff_m(Indent indent, const T &v1, const T &v2, std::string_view name) { + if (v1 == v2) { + return; + } + fmt::println("{}{}", indent, name); + diff(~indent, v1, v2); +} + +template +DIFF_F(jam::ConfigVec) { + diff(indent, v1.v, v2.v); +} + +template +DIFF_F(std::optional) { + if (v1 == v2) { + return; + } + std::string_view o[] = {"None", "Some"}; + auto i1 = v1 ? 1 : 0, i2 = v2 ? 1 : 0; + if (i1 != i2) { + fmt::println("{}{} != {}", indent, o[i1], o[i2]); + return; + } + fmt::println("{}{}", indent, o[i1]); + if (v1) { + diff(~indent, *v1, *v2); + } +} + +template + requires( + requires(T v) { std::span{v}; } + and not requires(T v) { qtils::BytesIn{v}; }) +void diff(Indent indent, const T &_v1, const T &_v2) { + if (_v1 == _v2) { + return; + } + std::span v1{_v1}, v2{_v2}; + if (v1.size() != v2.size()) { + fmt::println("{}{} != {}", indent, v1.size(), v2.size()); + return; + } + fmt::println("{}{}", indent, v1.size()); + for (size_t i = 0; i < v1.size(); ++i) { + if (v1[i] == v2[i]) { + continue; + } + fmt::println("{}[{}]", indent, i); + diff(~indent, v1[i], v2[i]); + } +} + +DIFF_F(qtils::BytesIn) { + if (v1.size() == v2.size() && memcmp(v1.data(), v2.data(), v1.size()) == 0) { + return; + } + diff1("{}- {:x}", indent, v1); + diff2("{}+ {:x}", indent, v2); +} diff --git a/test-vectors/history/.gitignore b/test-vectors/history/.gitignore new file mode 100644 index 0000000..4fc5df1 --- /dev/null +++ b/test-vectors/history/.gitignore @@ -0,0 +1,3 @@ +/types.hpp +/types.scale.hpp +/types.diff.hpp diff --git a/test-vectors/history/CMakeLists.txt b/test-vectors/history/CMakeLists.txt new file mode 100644 index 0000000..79a88a5 --- /dev/null +++ b/test-vectors/history/CMakeLists.txt @@ -0,0 +1,18 @@ +# +# Copyright Quadrivium LLC +# All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# + +asn1(history history) + +add_executable(test_vectors_history_test + history.test.cpp +) +target_compile_definitions(test_vectors_history_test PRIVATE PROJECT_SOURCE_DIR="${PROJECT_SOURCE_DIR}") +target_link_libraries(test_vectors_history_test + fmt::fmt + headers + test_vectors_history_types +) +add_test(test_vectors_history_test test_vectors_history_test) diff --git a/test-vectors/history/history.hpp b/test-vectors/history/history.hpp new file mode 100644 index 0000000..346ed34 --- /dev/null +++ b/test-vectors/history/history.hpp @@ -0,0 +1,68 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include + +namespace jam::history { + namespace types = test_vectors_history; + + /** + * The size of recent history, in blocks + */ + // [GP 0.4.3 I.4.4] + // https://github.com/gavofyork/graypaper/blob/v0.4.3/text/definitions.tex#L266 + constexpr uint32_t H = 8; + + // [GP 0.4.3 E.2 327] + // https://github.com/gavofyork/graypaper/blob/v0.4.3/text/merklization.tex#L205 + inline types::Mmr mathcal_A(types::Mmr r, types::Hash l) { + for (size_t n = 0; n < r.peaks.size(); ++n) { + if (not r.peaks[n]) { + r.peaks[n] = l; + return r; + } + auto r_n = qtils::optionTake(r.peaks[n]).value(); + l = mathcal_H_K(frown(r_n, l)); + } + r.peaks.emplace_back(l); + return r; + } + + /** + * Given state and input, derive next state and output. + */ + inline std::pair transition( + const types::State &state, const types::Input &input) { + // [GP 0.4.3 7 84] + // https://github.com/gavofyork/graypaper/blob/v0.4.3/text/recent_history.tex#L32 + std::vector beta_tick(state.beta.size() >= H ? std::next(state.beta.begin()) + : state.beta.begin(), + state.beta.end()); + // [GP 0.4.3 7 82] + // https://github.com/gavofyork/graypaper/blob/v0.4.3/text/recent_history.tex#L12 + if (not beta_tick.empty()) { + beta_tick.back().state_root = input.parent_state_root; + } + // [GP 0.4.3 7 83] + // https://github.com/gavofyork/graypaper/blob/v0.4.3/text/recent_history.tex#L20 + types::Mmr mmr; + if (not state.beta.empty()) { + mmr = state.beta.back().mmr; + } + auto mmr_tick = mathcal_A(mmr, input.accumulate_root); + beta_tick.emplace_back(types::BlockInfo{ + .header_hash = input.header_hash, + .mmr = mmr_tick, + .state_root = types::Hash{}, + .reported = input.work_packages, + }); + return std::make_pair(types::State{.beta = beta_tick}, types::Output{}); + } +} // namespace jam::history diff --git a/test-vectors/history/history.test.cpp b/test-vectors/history/history.test.cpp new file mode 100644 index 0000000..d73f9d6 --- /dev/null +++ b/test-vectors/history/history.test.cpp @@ -0,0 +1,47 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +using jam::test_vectors_history::Vectors; + +namespace types = jam::test_vectors_history; + +bool ok = true; + +/** + * Check history state transition against test vectors. + * @given `pre_state` + * @when transition with `input` + * @then get expected `post_state` and `output` + */ +void test_transition() { + Vectors vectors; + for (auto &path : vectors.paths) { + auto s = path.native(); + fmt::println("{}.json", s.substr(0, s.size() - 6)); + auto testcase = vectors.read(path); + auto [state, output] = + jam::history::transition(testcase.pre_state, testcase.input); + Indent indent{1}; + if (state != testcase.post_state) { + ok = false; + diff_m(indent, state, testcase.post_state, "state"); + } + if (output != testcase.output) { + ok = false; + diff_m(indent, output, testcase.output, "output"); + } + } +} + +int main() { + test_transition(); + fmt::println("ok"); + return 0; +} diff --git a/test-vectors/history/types.test.cpp b/test-vectors/history/types.test.cpp new file mode 100644 index 0000000..32e9b89 --- /dev/null +++ b/test-vectors/history/types.test.cpp @@ -0,0 +1,16 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +using jam::test_vectors::test_reencode; +using jam::test_vectors_history::Vectors; + +int main() { + test_reencode(); + fmt::println("ok"); + return 0; +} diff --git a/test-vectors/history/vectors.hpp b/test-vectors/history/vectors.hpp new file mode 100644 index 0000000..be0f2e5 --- /dev/null +++ b/test-vectors/history/vectors.hpp @@ -0,0 +1,18 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +namespace jam::test_vectors_history { + struct Vectors : test_vectors::VectorsT { + Vectors() : VectorsT{Config{}} { + this->list("history/data"); + } + }; +} // namespace jam::test_vectors_history diff --git a/test-vectors/jamtestvectors b/test-vectors/jamtestvectors index 15f1b2b..b9ba221 160000 --- a/test-vectors/jamtestvectors +++ b/test-vectors/jamtestvectors @@ -1 +1 @@ -Subproject commit 15f1b2b31fdf05aab1ec92c7190a9a302d6ae37a +Subproject commit b9ba221f4798ba8ebbfe64779e3d44ec1a35fe00 diff --git a/test-vectors/safrole/.gitignore b/test-vectors/safrole/.gitignore index cb56060..4fc5df1 100644 --- a/test-vectors/safrole/.gitignore +++ b/test-vectors/safrole/.gitignore @@ -1,2 +1,3 @@ /types.hpp /types.scale.hpp +/types.diff.hpp diff --git a/test-vectors/safrole/CMakeLists.txt b/test-vectors/safrole/CMakeLists.txt index c382151..6385f02 100644 --- a/test-vectors/safrole/CMakeLists.txt +++ b/test-vectors/safrole/CMakeLists.txt @@ -4,28 +4,4 @@ # SPDX-License-Identifier: Apache-2.0 # -set(SAFROLE_ASN_DIR ${PROJECT_SOURCE_DIR}/test-vectors/jamtestvectors/safrole) -set(SAFROLE_ASN ${SAFROLE_ASN_DIR}/safrole.asn ${SAFROLE_ASN_DIR}/tiny.asn ${SAFROLE_ASN_DIR}/full.asn) -set(SAFROLE_OUTPUT_PREFIX ${PROJECT_SOURCE_DIR}/test-vectors/safrole/types) -set(SAFROLE_OUTPUT ${SAFROLE_OUTPUT_PREFIX}.hpp ${SAFROLE_OUTPUT_PREFIX}.scale.hpp) -set(SAFROLE_PY ${PROJECT_SOURCE_DIR}/python/asn1.py) -add_custom_command( - OUTPUT ${SAFROLE_OUTPUT} - COMMAND ${Python3_EXECUTABLE} ${SAFROLE_PY} - DEPENDS ${SAFROLE_PY} ${SAFROLE_ASN} -) - -add_library(test_vectors_safrole_types INTERFACE ${SAFROLE_OUTPUT}) -target_link_libraries(test_vectors_safrole_types INTERFACE - scale::scale -) - -add_executable(test_vectors_safrole_types_test - types.test.cpp -) -target_compile_definitions(test_vectors_safrole_types_test PRIVATE PROJECT_SOURCE_DIR="${PROJECT_SOURCE_DIR}") -target_link_libraries(test_vectors_safrole_types_test - fmt::fmt - test_vectors_safrole_types -) -add_test(NAME test_vectors_safrole_types_test COMMAND $) +asn1(safrole safrole tiny full) diff --git a/test-vectors/safrole/types.test.cpp b/test-vectors/safrole/types.test.cpp index 466d72f..256aace 100644 --- a/test-vectors/safrole/types.test.cpp +++ b/test-vectors/safrole/types.test.cpp @@ -4,30 +4,14 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "vectors.hpp" +#include +using jam::test_vectors::test_reencode; using jam::test_vectors_safrole::Vectors; -/** - * Check python generated scale encoding/decoding against test vectors. - */ -template -void test_reencode() { - Vectors vectors; - for (auto &path : vectors.paths) { - fmt::println("{}", path.native()); - auto expected = vectors.readRaw(path); - auto decoded = vectors.decode(expected); - auto reencoded = scale::encode(*decoded).value(); - if (reencoded != expected) { - throw std::logic_error{"reencoded"}; - } - } -} - int main() { - test_reencode(); - test_reencode(); + test_reencode>(); + test_reencode>(); fmt::println("ok"); return 0; } diff --git a/test-vectors/safrole/vectors.hpp b/test-vectors/safrole/vectors.hpp index 3f5b731..8ebe447 100644 --- a/test-vectors/safrole/vectors.hpp +++ b/test-vectors/safrole/vectors.hpp @@ -4,42 +4,18 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include -#include +#pragma once -#include "types.scale.hpp" +#include +#include namespace jam::test_vectors_safrole { template - struct Vectors { - using types = std::conditional_t; + struct Vectors : test_vectors::VectorsT { + static constexpr std::string_view type = is_full ? "full" : "tiny"; - std::string type = is_full ? "full" : "tiny"; - std::filesystem::path dir = std::filesystem::path{PROJECT_SOURCE_DIR} - / "test-vectors/jamtestvectors/safrole" / type; - std::set paths; - - Vectors() { - for (auto &file : std::filesystem::directory_iterator{dir}) { - if (file.path().extension() == ".scale") { - paths.emplace(file.path()); - } - } - } - - static auto decode(qtils::BytesIn raw) { - scale::ScaleDecoderStream s{raw}; - auto testcase = std::make_unique(); - s >> *testcase; - return testcase; - } - - static auto readRaw(const std::filesystem::path &path) { - return qtils::readBytes(path).value(); - } - - static auto read(const std::filesystem::path &path) { - return decode(readRaw(path)); + Vectors() : VectorsT{is_full ? config_full : config_tiny} { + this->list(std::filesystem::path{"safrole"} / type); } }; } // namespace jam::test_vectors_safrole diff --git a/test-vectors/vectors.hpp b/test-vectors/vectors.hpp new file mode 100644 index 0000000..aea195c --- /dev/null +++ b/test-vectors/vectors.hpp @@ -0,0 +1,86 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include + +/** + * Common functions for test vectors + */ + +namespace jam::test_vectors { + inline const std::filesystem::path dir = + std::filesystem::path{PROJECT_SOURCE_DIR} / "test-vectors/jamtestvectors"; + + template + struct VectorsT { + Config config; + std::set paths; + + VectorsT(Config config) : config{config} {} + + void list(const std::filesystem::path &relative) { + auto ext_scale = ".scale", ext_json = ".json"; + auto use_ext = ext_scale; + std::map path_ok; + for (auto &file : std::filesystem::directory_iterator{dir / relative}) { + auto path = file.path(), ext = path.extension(); + if (ext != ext_scale and ext != ext_json) { + continue; + } + path.replace_extension(use_ext); + auto &ok = path_ok[path]; + ok = ok or ext == use_ext; + } + for (auto &[path, ok] : path_ok) { + if (not ok) { + fmt::println( + "{}:{} warning: {} is missing, but files with other extensions are available", + __FILE__, + __LINE__, + path.native()); + continue; + } + paths.emplace(path); + } + } + + auto decode(qtils::BytesIn raw) const { + scale::ScaleDecoderStream s{raw}; + T testcase; + decodeConfig(s, testcase, config); + return testcase; + } + + static auto readRaw(const std::filesystem::path &path) { + return qtils::readBytes(path).value(); + } + + auto read(const std::filesystem::path &path) const { + return decode(readRaw(path)); + } + }; + + /** + * Check python generated scale encoding/decoding against test vectors. + */ + template + void test_reencode(const T &vectors = {}) { + for (auto &path : vectors.paths) { + fmt::println("{}", path.native()); + auto expected = vectors.readRaw(path); + auto decoded = vectors.decode(expected); + auto reencoded = scale::encode(decoded).value(); + if (reencoded != expected) { + throw std::logic_error{"reencoded"}; + } + } + } +} // namespace jam::test_vectors diff --git a/vcpkg.json b/vcpkg.json index d044aa4..3eaec59 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -2,10 +2,7 @@ "name": "cpp-jam", "version": "0.0.1", "dependencies": [ - "scale", - { - "name": "fmt", - "version>=": "10.2.1#2" - } + "fmt", + "scale" ] }