diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..5c1d881 --- /dev/null +++ b/.clang-format @@ -0,0 +1,27 @@ +# +# Copyright Quadrivium LLC +# All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# + +BasedOnStyle: Google + +AlignAfterOpenBracket: DontAlign +AlignOperands: DontAlign +AllowShortFunctionsOnASingleLine: Empty +BinPackArguments: false +BinPackParameters: false +BreakBeforeBinaryOperators: NonAssignment +BreakStringLiterals: false +DerivePointerAlignment: false +FixNamespaceComments: true +IncludeBlocks: Preserve +InsertBraces: true +InsertTrailingCommas: Wrapped +NamespaceIndentation: All +PointerAlignment: Right +QualifierAlignment: Custom +QualifierOrder: ['inline', 'static', 'constexpr', 'const', 'volatile', 'type', 'restrict'] +ReferenceAlignment: Right +ShortNamespaceLines: 0 +SortIncludes: CaseSensitive diff --git a/.gitignore b/.gitignore index 259148f..20b2df6 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ *.exe *.out *.app + +/build diff --git a/CMakeLists.txt b/CMakeLists.txt index a7b62fc..7ecad11 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,11 +10,21 @@ project(cpp-jam VERSION 0.0.1 LANGUAGES C CXX ) +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) option(TESTING "Build and run test suite" ON) +find_package(Python3 REQUIRED) + +find_package(Boost CONFIG REQUIRED) +find_package(fmt CONFIG REQUIRED) +find_package(scale CONFIG REQUIRED) + add_subdirectory(src) if(TESTING) + enable_testing() add_subdirectory(test-vectors) endif () diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 0000000..69e9a20 --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,14 @@ +{ + "version": 2, + "configurePresets": [ + { + "name": "default", + "generator": "Ninja", + "binaryDir": "${sourceDir}/build", + "cacheVariables": { + "CMAKE_TOOLCHAIN_FILE": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake", + "CMAKE_BUILD_TYPE": "Debug" + } + } + ] +} diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..425663e --- /dev/null +++ b/python/README.md @@ -0,0 +1,9 @@ +create venv\ +`uv venv`\ +`source .venv/bin/activate` + +install dependencies using [uv](https://github.com/astral-sh/uv)\ +`uv pip install -r requirements.txt` + +format python code using [ruff](https://github.com/astral-sh/ruff)\ +`ruff format asn1.py` diff --git a/python/asn1.py b/python/asn1.py new file mode 100755 index 0000000..6a70a7d --- /dev/null +++ b/python/asn1.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 + +import asn1tools +import os.path + +DIR = os.path.dirname(__file__) +TEST_VECTORS_DIR = os.path.join(DIR, "../test-vectors") +OUTPUT_NAME = "types" +OUTPUT_DIR = os.path.join(TEST_VECTORS_DIR, "safrole") +SAFROLE_DIR = os.path.join(TEST_VECTORS_DIR, "jamtestvectors/safrole") +ARGS = ["validators-count", "epoch-length"] +NS = "jam::test_vectors_safrole" + + +def flatten(aaa: list[list]): + return [a for aa in aaa for a in aa] + + +def safrole_file(name: str): + return os.path.join(SAFROLE_DIR, name + ".asn") + + +def write(ext: str, lines: list[str]): + with open(os.path.join(OUTPUT_DIR, OUTPUT_NAME + ext), "w") as f: + f.writelines(line + "\n" for line in lines) + + +def c_using(name: str, other: str): + return ["using %s = %s;" % (name, other)] + + +def c_struct(name: str, members): + return [ + "struct %s {" % name, + *((" %s %s;" % (t, k) for k, t in members)), + " bool operator==(const %s &)const = default;" % name, + "};", + ] + + +def c_dash(s: int | str): + if isinstance(s, int): + return s + return s.replace("-", "_") + + +def indent(lines): + return [" " + x for x in lines] + + +class Type: + def __init__(self, name: str): + self.name = name + self.args: list[str] = [] + self.decl: list[str] = [] + self.scale: list[str] = [] + + def c_tdecl(self): + if not self.args: + return [] + return ["template<%s>" % ", ".join("uint32_t %s" % c_dash(x) for x in self.args)] + + def c_targs(self): + if not self.args: + return "" + return "<%s>" % ", ".join(c_dash(x) for x in self.args) + + def c_tname(self): + return self.name + self.c_targs() + + +def asn_recurse(t: dict): + yield t + e = t.get("element", None) + if e: + yield from asn_recurse(e) + m = t.get("members", None) + if m: + for x in m: + yield from asn_recurse(x) + + +def asn_deps(types: dict): + def deep(k: str): + yield k + for dep in deps1[k]: + yield from deep(dep) + + deps1 = { + k: {x for x in (x["type"] for x in asn_recurse(t)) if x in types} + for k, t in types.items() + } + deps1 = {k: set(flatten(map(deep, deps))) for k, deps in deps1.items()} + deps2: dict[str, set[Type]] = {k: set() for k in types} + for k, deps in deps1.items(): + for dep in deps: + deps2[dep].add(k) + return deps1, deps2 + + +def asn_args(types: dict, deps2: dict[str, set[str]]): + args = { + k: { + s[0] + for s in (x.get("size", None) for x in asn_recurse(t)) + if s and isinstance(s[0], str) + } + for k, t in types.items() + } + for tname, deps in deps2.items(): + for dep in deps: + args[dep].update(args[tname]) + return {k: [a for a in ARGS if a in aa] for k, aa in args.items()} + + +def c_scale(ty: Type, encode: list[str], decode: list[str]): + return [ + *ty.c_tdecl(), + "inline scale::ScaleEncoderStream &operator<<(scale::ScaleEncoderStream &s, const %s &v) {" + % ty.c_tname(), + *indent(encode), + " return s;", + "}", + *ty.c_tdecl(), + "inline scale::ScaleDecoderStream &operator>>(scale::ScaleDecoderStream &s, %s &v) {" + % ty.c_tname(), + *indent(decode), + " return s;", + "}", + ] + + +def c_scale_struct(ty: Type, members: list[str]): + return c_scale( + ty, + ["s << v.%s;" % x for x in members], + ["s >> v.%s;" % x for x in members], + ) + + +def parse_types(path: str): + def asn_sequence_of(t): + (size,) = t["size"] + fixed = isinstance(size, (int, str)) + T = t["element"]["type"] + assert T in asn_types + if T == "U8": + if fixed: + return "qtils::BytesN<%s>" % c_dash(size) + return "qtils::Bytes" + if fixed: + return "std::array<%s, %s>" % (T, c_dash(size)) + return "std::vector<%s>" % T + + def asn_member(t): + if t["type"] == "SEQUENCE OF": + r = asn_sequence_of(t) + elif t["type"] in asn_types: + r = types[t["type"]].c_tname() + else: + raise TypeError(t) + if t.get("optional", False): + return "std::optional<%s>" % r + return r + + asn_types: dict = asn1tools.parse_files([path])["SafroleModule"]["types"] + deps1, deps2 = asn_deps(asn_types) + types = {tname: Type(tname) for tname in asn_types} + for tname, args in asn_args(asn_types, deps2).items(): + types[tname].args = args + enum_trait = [] + for tname, t in asn_types.items(): + ty = types[tname] + if tname == "U8": + ty.decl = c_using(tname, "uint8_t") + continue + if tname == "U32": + ty.decl = c_using(tname, "uint32_t") + continue + if t["type"] == "CHOICE": + ty.decl = c_struct( + tname, + [ + ( + "v", + "boost::variant<%s>" + % ", ".join(asn_member(x) for x in t["members"]), + ) + ], + ) + ty.scale.extend(c_scale_struct(ty, ["v"])) + continue + if t["type"] == "ENUMERATED": + values = [x[1] for x in t["values"]] + assert all(x == i and x < 0xFF for i, x in enumerate(values)) + ty.decl = [ + "enum class %s : uint8_t {" % tname, + *(" %s," % c_dash(x[0]) for x in t["values"]), + "};", + ] + ns = "%s::generic" % NS + enum_trait.append( + "SCALE_DEFINE_ENUM_VALUE_LIST(%s, %s, %s)" + % ( + ns, + ty.name, + ", ".join( + "%s::%s::%s" % (ns, ty.name, c_dash(x[0])) for x in t["values"] + ), + ) + ) + continue + if t["type"] == "SEQUENCE": + ty.decl = c_struct( + tname, [(c_dash(x["name"]), asn_member(x)) for x in t["members"]] + ) + ty.scale.extend( + c_scale_struct(ty, [c_dash(x["name"]) for x in t["members"]]) + ) + continue + ty.decl = c_using(tname, asn_member(t)) + + order = asn1tools.c.utils.topological_sort(deps1) + return [types[k] for k in order], enum_trait + + +def parse_const(path: str): + values: dict = asn1tools.parse_files([path])["SafroleConstants"]["values"] + assert all(v["type"] == "INTEGER" for v in values.values()) + return {k: v["value"] for k, v in values.items()} + + +types, enum_trait = parse_types(safrole_file("safrole")) + + +g_types = flatten([*ty.c_tdecl(), *ty.decl] for ty in types) +g_types = ["namespace %s::generic {" % NS, *indent(g_types), "}"] +for name in ["tiny", "full"]: + args = parse_const(safrole_file(name)) + g_args = [ + *["static constexpr uint32_t %s = %s;" % (c_dash(a), args[a]) for a in ARGS], + *flatten( + c_using(ty.name, "%s::generic::%s" % (NS, ty.c_tname())) for ty in types + ), + ] + g_args = ["struct %s {" % name, *indent(g_args), "};"] + g_types.extend( + [ + "namespace %s {" % NS, + *indent(g_args), + "}", + ] + ) +g_types = [ + "#pragma once", + "#include ", + "#include ", + *g_types, +] + +g_scale = flatten(ty.scale for ty in types) +g_scale = ["namespace %s::generic {" % NS, *indent(g_scale), "}"] +g_scale = [ + "#pragma once", + "#include ", + '#include "%s.hpp"' % OUTPUT_NAME, + *g_scale, + *enum_trait, +] + +write(".hpp", g_types) +write(".scale.hpp", g_scale) diff --git a/python/requirements.txt b/python/requirements.txt new file mode 100644 index 0000000..1dc2202 --- /dev/null +++ b/python/requirements.txt @@ -0,0 +1 @@ +asn1tools diff --git a/test-vectors/CMakeLists.txt b/test-vectors/CMakeLists.txt index 4fd9231..bf96af0 100644 --- a/test-vectors/CMakeLists.txt +++ b/test-vectors/CMakeLists.txt @@ -3,3 +3,5 @@ # All Rights Reserved # SPDX-License-Identifier: Apache-2.0 # + +add_subdirectory(safrole) diff --git a/test-vectors/safrole/.gitignore b/test-vectors/safrole/.gitignore new file mode 100644 index 0000000..cb56060 --- /dev/null +++ b/test-vectors/safrole/.gitignore @@ -0,0 +1,2 @@ +/types.hpp +/types.scale.hpp diff --git a/test-vectors/safrole/CMakeLists.txt b/test-vectors/safrole/CMakeLists.txt index 4fd9231..c382151 100644 --- a/test-vectors/safrole/CMakeLists.txt +++ b/test-vectors/safrole/CMakeLists.txt @@ -3,3 +3,29 @@ # All Rights Reserved # SPDX-License-Identifier: Apache-2.0 # + +set(SAFROLE_ASN_DIR ${PROJECT_SOURCE_DIR}/test-vectors/jamtestvectors/safrole) +set(SAFROLE_ASN ${SAFROLE_ASN_DIR}/safrole.asn ${SAFROLE_ASN_DIR}/tiny.asn ${SAFROLE_ASN_DIR}/full.asn) +set(SAFROLE_OUTPUT_PREFIX ${PROJECT_SOURCE_DIR}/test-vectors/safrole/types) +set(SAFROLE_OUTPUT ${SAFROLE_OUTPUT_PREFIX}.hpp ${SAFROLE_OUTPUT_PREFIX}.scale.hpp) +set(SAFROLE_PY ${PROJECT_SOURCE_DIR}/python/asn1.py) +add_custom_command( + OUTPUT ${SAFROLE_OUTPUT} + COMMAND ${Python3_EXECUTABLE} ${SAFROLE_PY} + DEPENDS ${SAFROLE_PY} ${SAFROLE_ASN} +) + +add_library(test_vectors_safrole_types INTERFACE ${SAFROLE_OUTPUT}) +target_link_libraries(test_vectors_safrole_types INTERFACE + scale::scale +) + +add_executable(test_vectors_safrole_types_test + types.test.cpp +) +target_compile_definitions(test_vectors_safrole_types_test PRIVATE PROJECT_SOURCE_DIR="${PROJECT_SOURCE_DIR}") +target_link_libraries(test_vectors_safrole_types_test + fmt::fmt + test_vectors_safrole_types +) +add_test(NAME test_vectors_safrole_types_test COMMAND $) diff --git a/test-vectors/safrole/types.test.cpp b/test-vectors/safrole/types.test.cpp new file mode 100644 index 0000000..466d72f --- /dev/null +++ b/test-vectors/safrole/types.test.cpp @@ -0,0 +1,33 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "vectors.hpp" + +using jam::test_vectors_safrole::Vectors; + +/** + * Check python generated scale encoding/decoding against test vectors. + */ +template +void test_reencode() { + Vectors vectors; + for (auto &path : vectors.paths) { + fmt::println("{}", path.native()); + auto expected = vectors.readRaw(path); + auto decoded = vectors.decode(expected); + auto reencoded = scale::encode(*decoded).value(); + if (reencoded != expected) { + throw std::logic_error{"reencoded"}; + } + } +} + +int main() { + test_reencode(); + test_reencode(); + fmt::println("ok"); + return 0; +} diff --git a/test-vectors/safrole/vectors.hpp b/test-vectors/safrole/vectors.hpp new file mode 100644 index 0000000..3f5b731 --- /dev/null +++ b/test-vectors/safrole/vectors.hpp @@ -0,0 +1,45 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +#include "types.scale.hpp" + +namespace jam::test_vectors_safrole { + template + struct Vectors { + using types = std::conditional_t; + + std::string type = is_full ? "full" : "tiny"; + std::filesystem::path dir = std::filesystem::path{PROJECT_SOURCE_DIR} + / "test-vectors/jamtestvectors/safrole" / type; + std::set paths; + + Vectors() { + for (auto &file : std::filesystem::directory_iterator{dir}) { + if (file.path().extension() == ".scale") { + paths.emplace(file.path()); + } + } + } + + static auto decode(qtils::BytesIn raw) { + scale::ScaleDecoderStream s{raw}; + auto testcase = std::make_unique(); + s >> *testcase; + return testcase; + } + + static auto readRaw(const std::filesystem::path &path) { + return qtils::readBytes(path).value(); + } + + static auto read(const std::filesystem::path &path) { + return decode(readRaw(path)); + } + }; +} // namespace jam::test_vectors_safrole diff --git a/vcpkg-configuration.json b/vcpkg-configuration.json new file mode 100644 index 0000000..1ad42d2 --- /dev/null +++ b/vcpkg-configuration.json @@ -0,0 +1,17 @@ +{ + "default-registry": { + "kind": "git", + "baseline": "fe1cde61e971d53c9687cf9a46308f8f55da19fa", + "repository": "https://github.com/microsoft/vcpkg" + }, + "registries": [ + { + "kind": "artifact", + "location": "https://github.com/microsoft/vcpkg-ce-catalog/archive/refs/heads/main.zip", + "name": "microsoft" + } + ], + "overlay-ports": [ + "vcpkg-overlay" + ] +} diff --git a/vcpkg-overlay/qtils/portfile.cmake b/vcpkg-overlay/qtils/portfile.cmake new file mode 100644 index 0000000..9f91a98 --- /dev/null +++ b/vcpkg-overlay/qtils/portfile.cmake @@ -0,0 +1,11 @@ +vcpkg_check_linkage(ONLY_STATIC_LIBRARY) +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO qdrvm/qtils + REF 665971b1774b0b3e6b0bdba864480e85a5c7d1d5 + SHA512 23f443e48c66d8c3a20c419597c24717ef330f967b81ba3c7ad90575c53a6e321cd7d5ecd629ce772b1852f72c43a0b2e3d4a0ab1f7776a8ec9c7294b82b3c3d +) +vcpkg_cmake_configure(SOURCE_PATH "${SOURCE_PATH}") +vcpkg_cmake_install() +vcpkg_cmake_config_fixup(PACKAGE_NAME "qtils") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") diff --git a/vcpkg-overlay/qtils/vcpkg.json b/vcpkg-overlay/qtils/vcpkg.json new file mode 100644 index 0000000..96668a6 --- /dev/null +++ b/vcpkg-overlay/qtils/vcpkg.json @@ -0,0 +1,11 @@ +{ + "name": "qtils", + "version": "0.0.3", + "dependencies": [ + { "name": "vcpkg-cmake", "host": true }, + { "name": "vcpkg-cmake-config", "host": true }, + "boost-algorithm", + "boost-outcome", + "fmt" + ] +} diff --git a/vcpkg-overlay/scale/portfile.cmake b/vcpkg-overlay/scale/portfile.cmake new file mode 100644 index 0000000..4df6b84 --- /dev/null +++ b/vcpkg-overlay/scale/portfile.cmake @@ -0,0 +1,11 @@ +vcpkg_check_linkage(ONLY_STATIC_LIBRARY) +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO qdrvm/scale-codec-cpp + REF ab9fb76632aa8ae98978ce3c54ed88a2fd29fd4e + SHA512 51e013a7a6e40f30f64a490251c37636b13566660c43a5245c5e5c1c3e4de3f18b81b730cacbaccfc5dd9d64b82ddec1ebcf04a47089a9be2353c82a3f3b0f25 +) +vcpkg_cmake_configure(SOURCE_PATH "${SOURCE_PATH}") +vcpkg_cmake_install() +vcpkg_cmake_config_fixup(PACKAGE_NAME "scale") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") diff --git a/vcpkg-overlay/scale/vcpkg.json b/vcpkg-overlay/scale/vcpkg.json new file mode 100644 index 0000000..e58f73f --- /dev/null +++ b/vcpkg-overlay/scale/vcpkg.json @@ -0,0 +1,12 @@ +{ + "name": "scale", + "version": "1.1.3", + "dependencies": [ + { "name": "vcpkg-cmake", "host": true }, + { "name": "vcpkg-cmake-config", "host": true }, + "boost-endian", + "boost-multiprecision", + "boost-variant", + "qtils" + ] +} diff --git a/vcpkg.json b/vcpkg.json new file mode 100644 index 0000000..d044aa4 --- /dev/null +++ b/vcpkg.json @@ -0,0 +1,11 @@ +{ + "name": "cpp-jam", + "version": "0.0.1", + "dependencies": [ + "scale", + { + "name": "fmt", + "version>=": "10.2.1#2" + } + ] +}