diff --git a/CMakeLists.txt b/CMakeLists.txt index a33cc47..2fd5049 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) if (MSVC) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /Gm- /W3 /WX /EHsc /GS /fp:precise /permissive- /Zc:wchar_t /Zc:forScope /Zc:inline /Gd /TP") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /D_CRT_SECURE_NO_WARNINGS /FC") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /D_CRT_SECURE_NO_WARNINGS /FC /wd4244 /wd4267") set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /MDd") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /O2 /Oi /Gy- /MD") diff --git a/examples/checkpoint_recovery-dir/checkpoint_recovery.cc b/examples/checkpoint_recovery-dir/checkpoint_recovery.cc index f95d2cc..4c23d87 100644 --- a/examples/checkpoint_recovery-dir/checkpoint_recovery.cc +++ b/examples/checkpoint_recovery-dir/checkpoint_recovery.cc @@ -219,12 +219,12 @@ int main(int argc, char* argv[]) { store.StartSession(); auto lib_id = store.LoadPSFLibrary(argv[2]); - auto id_proj = store.MakeProjection("id"); - auto actor_id_proj = store.MakeProjection("actor.id"); - auto repo_id_proj = store.MakeProjection("repo.id"); - auto type_proj = store.MakeProjection("type"); - auto predicate1_id = store.MakeInlinePSF({ "type", "payload.action" }, lib_id, "opened_issue"); - auto predicate2_id = store.MakeInlinePSF({ "type", "payload.pull_request.head.repo.language" }, lib_id, "cpp_pr"); + auto id_proj = store.MakeProjection("/id"); + auto actor_id_proj = store.MakeProjection("/actor/id"); + auto repo_id_proj = store.MakeProjection("/repo/id"); + auto type_proj = store.MakeProjection("/type"); + auto predicate1_id = store.MakeInlinePSF({ "/type", "/payload/action" }, lib_id, "opened_issue"); + auto predicate2_id = store.MakeInlinePSF({ "/type", "/payload/pull_request/head/repo/language" }, lib_id, "cpp_pr"); std::vector parser_actions; parser_actions.push_back({ REGISTER_GENERAL_PSF, id_proj }); diff --git a/examples/github_query-dir/github_query.cc b/examples/github_query-dir/github_query.cc index 6574214..ae8033b 100644 --- a/examples/github_query-dir/github_query.cc +++ b/examples/github_query-dir/github_query.cc @@ -206,14 +206,14 @@ int main(int argc, char* argv[]) { store.StartSession(); auto lib_id = store.LoadPSFLibrary(argv[2]); - auto id_proj = store.MakeProjection("id"); - auto actor_id_proj = store.MakeInlinePSF({ "actor.id" }, lib_id, "int_projection"); - auto repo_id_proj = store.MakeInlinePSF({ "repo.id" }, lib_id, "int_projection"); - auto type_proj = store.MakeProjection("type"); + auto id_proj = store.MakeProjection("/id"); + auto actor_id_proj = store.MakeInlinePSF({ "/actor/id" }, lib_id, "int_projection"); + auto repo_id_proj = store.MakeInlinePSF({ "/repo/id" }, lib_id, "int_projection"); + auto type_proj = store.MakeProjection("/type"); auto predicate1_id = - store.MakeInlinePSF({ "type", "payload.action" }, lib_id, "opened_issue"); + store.MakeInlinePSF({ "/type", "/payload/action" }, lib_id, "opened_issue"); auto predicate2_id = - store.MakeInlinePSF({ "type", "payload.pull_request.head.repo.language" }, lib_id, "cpp_pr"); + store.MakeInlinePSF({ "/type", "/payload/pull_request/head.repo/language" }, lib_id, "cpp_pr"); std::vector parser_actions; parser_actions.push_back({ REGISTER_GENERAL_PSF, id_proj }); diff --git a/examples/twitter_query-dir/twitter_query.cc b/examples/twitter_query-dir/twitter_query.cc index 897ef5c..56f4956 100644 --- a/examples/twitter_query-dir/twitter_query.cc +++ b/examples/twitter_query-dir/twitter_query.cc @@ -206,15 +206,15 @@ int main(int argc, char* argv[]) { store.StartSession(); auto lib_id = store.LoadPSFLibrary(argv[2]); - auto id_proj = store.MakeProjection("id"); - auto user_id_proj = store.MakeProjection("user.id"); - auto reply_status_id_proj = store.MakeProjection("in_reply_to_status_id"); - auto reply_user_id_proj = store.MakeProjection("in_reply_to_user_id"); - auto lang_proj = store.MakeProjection("lang"); + auto id_proj = store.MakeProjection("/id"); + auto user_id_proj = store.MakeProjection("/user/id"); + auto reply_status_id_proj = store.MakeProjection("/in_reply_to_status_id"); + auto reply_user_id_proj = store.MakeProjection("/in_reply_to_user_id"); + auto lang_proj = store.MakeProjection("/lang"); auto predicate1_id = - store.MakeInlinePSF({ "user.lang", "user.followers_count" }, lib_id, "jp_high_follower"); + store.MakeInlinePSF({ "/user/lang", "/user/followers_count" }, lib_id, "jp_high_follower"); auto predicate2_id = - store.MakeInlinePSF({ "in_reply_to_screen_name", "possibly_sensitive" }, lib_id, "sensitive_reply_to_trump"); + store.MakeInlinePSF({ "/in_reply_to_screen_name", "/possibly_sensitive" }, lib_id, "sensitive_reply_to_trump"); std::vector parser_actions; parser_actions.push_back({ REGISTER_GENERAL_PSF, id_proj }); diff --git a/src/adapters/simdjson_adapter.h b/src/adapters/simdjson_adapter.h index a56d158..23651c3 100644 --- a/src/adapters/simdjson_adapter.h +++ b/src/adapters/simdjson_adapter.h @@ -8,27 +8,21 @@ #include #include +#ifdef _MSC_VER +#define NOMINMAX +#endif #include #include #include "adapters/common_utils.h" +using namespace simdjson; + namespace fishstore { namespace adapter { -struct TreeNode { - TreeNode(): field_id(-1) {} - TreeNode(int64_t id): field_id(id) {} - ~TreeNode() { - for (auto x: children) delete x.second; - } - - int64_t field_id = -1; - std::unordered_map children; -}; - class SIMDJsonField { public: - SIMDJsonField(int64_t id_, const ParsedJson::iterator& it_) + SIMDJsonField(int64_t id_, const ParsedJson::Iterator& it_) : field_id(id_), iter(it_) {} inline int64_t FieldId() const { @@ -84,7 +78,7 @@ class SIMDJsonField { private: int64_t field_id; - ParsedJson::iterator iter; + ParsedJson::Iterator iter; }; class SIMDJsonRecord { @@ -113,46 +107,18 @@ class SIMDJsonRecord { class SIMDJsonParser { public: - SIMDJsonParser(const std::vector& field_names, const size_t alloc_bytes = 1LL << 25) { - root = new TreeNode{}; - for (auto field_id = 0; field_id < field_names.size(); ++field_id) { - auto& field_name = field_names[field_id]; - auto current_tree_node = root; - size_t start_pos = 0, pos = 0; - while (std::string::npos != (pos = field_name.find('.', start_pos))) { - auto slice = field_name.substr(start_pos, pos - start_pos); - auto iter = current_tree_node->children.find(slice); - if (iter != current_tree_node->children.end()) { - current_tree_node = iter->second; - } - else { - auto new_node = current_tree_node->children.emplace(std::make_pair(slice, new TreeNode{})); - current_tree_node = new_node.first->second; - } - start_pos = pos + 1; - } - - auto slice = field_name.substr(start_pos); - auto iter = current_tree_node->children.find(slice); - if (iter != current_tree_node->children.end()) { - printf("Skipping field %s as seen before...\n", field_name.c_str()); - } - else current_tree_node->children.emplace(std::make_pair(slice, new TreeNode{ field_id })); - } - auto success = pj.allocateCapacity(alloc_bytes); + SIMDJsonParser(const std::vector& field_names, const size_t alloc_bytes = 1LL << 25) + : fields(field_names) { + auto success = pj.allocate_capacity(alloc_bytes); assert(success); has_next = false; } - ~SIMDJsonParser() { - delete root; - } - inline void Load(const char* buffer, size_t length) { record.original = StringRef(buffer, length); record.fields.clear(); auto ok = json_parse(buffer, length, pj); - if (ok != 0 || !pj.isValid()) { + if (ok != 0 || !pj.is_valid()) { printf("Parsing failed...\n"); has_next = false; } else { @@ -165,35 +131,18 @@ class SIMDJsonParser { } inline const SIMDJsonRecord& NextRecord() { - ParsedJson::iterator it(pj); - Traverse(it, root, record.fields); + ParsedJson::Iterator it(pj); + for (auto field_id = 0; field_id < fields.size(); ++field_id) { + if (it.move_to(fields[field_id])) { + record.fields.emplace_back(SIMDJsonField{field_id, it}); + } + } has_next = false; return record; } private: - void Traverse(ParsedJson::iterator& it, const TreeNode* current_tree_node, - std::vector& parsed_fields) { - - if (current_tree_node->field_id != -1) { - parsed_fields.emplace_back(SIMDJsonField(current_tree_node->field_id, it)); - } - if (!current_tree_node->children.empty() && it.is_object()) { - if (it.down()) { - do { - std::string slice(it.get_string()); - auto slice_iter = current_tree_node->children.find(slice); - it.next(); - if (slice_iter != current_tree_node->children.end()) { - Traverse(it, slice_iter->second, parsed_fields); - } - } while (it.next()); - it.up(); - } - } - } - - TreeNode* root; + std::vector fields; ParsedJson pj; SIMDJsonRecord record; bool has_next; diff --git a/src/core/fishstore.h b/src/core/fishstore.h index 9146e27..e6582fd 100644 --- a/src/core/fishstore.h +++ b/src/core/fishstore.h @@ -15,6 +15,7 @@ #include #include #include +#include #include "device/file_system_disk.h" #include "tsl/hopscotch_map.h" @@ -1403,7 +1404,7 @@ inline OperationStatus FishStore::InternalScan(C& pending_context, uint64_t latest_record_version = 0; const KeyPointer* kpt = reinterpret_cast(hlog.Get(address)); - while(address >= end_addr && address >= head_address) { + while(address > end_addr && address >= head_address) { address = kpt->prev_address; kpt = reinterpret_cast(hlog.Get(address)); } @@ -1482,7 +1483,7 @@ inline OperationStatus FishStore::InternalFullScan(C& pending_context, if(current_page < hlog.head_address.load().page()) break; // Scan the page, record by record to the end of the page or current tail. - while(address <= to_address) { + while(address < to_address) { record_t* record = reinterpret_cast(hlog.Get(address)); if(record->header.IsNull()) { address += sizeof(RecordInfo); diff --git a/test/checkpoint_test.h b/test/checkpoint_test.h index f87ada0..cfe9384 100644 --- a/test/checkpoint_test.h +++ b/test/checkpoint_test.h @@ -165,8 +165,8 @@ TEST(CLASS, Checkpoint_Concurrent) { { store_t store{ 8192, 201326592, "test" }; store.StartSession(); - auto id_proj = store.MakeProjection("id"); - auto gender_proj = store.MakeProjection("gender"); + auto id_proj = store.MakeProjection("/id"); + auto gender_proj = store.MakeProjection("/gender"); std::vector actions; actions.push_back({ REGISTER_GENERAL_PSF, id_proj }); actions.push_back({ REGISTER_GENERAL_PSF, gender_proj }); diff --git a/test/in_memory_test.cc b/test/in_memory_test.cc index 15370da..6a4f1fa 100644 --- a/test/in_memory_test.cc +++ b/test/in_memory_test.cc @@ -160,8 +160,8 @@ class JsonFullScanContext : public IAsyncContext { TEST(InMemFishStore, Ingest_Serial) { store_t store{ 1LL << 24, 1LL << 30, "" }; store.StartSession(); - auto id_proj = store.MakeProjection("id"); - auto gender_proj = store.MakeProjection("gender"); + auto id_proj = store.MakeProjection("/id"); + auto gender_proj = store.MakeProjection("/gender"); std::vector actions; actions.push_back({ REGISTER_GENERAL_PSF, id_proj }); actions.push_back({ REGISTER_GENERAL_PSF, gender_proj }); @@ -213,8 +213,8 @@ TEST(InMemFishStore, Ingest_Serial) { TEST(InMemFishStore, Ingest_Concurrent) { store_t store{ 1LL << 24, 1LL << 30, "" }; store.StartSession(); - auto id_proj = store.MakeProjection("id"); - auto gender_proj = store.MakeProjection("gender"); + auto id_proj = store.MakeProjection("/id"); + auto gender_proj = store.MakeProjection("/gender"); std::vector actions; actions.push_back({ REGISTER_GENERAL_PSF, id_proj }); actions.push_back({ REGISTER_GENERAL_PSF, gender_proj }); @@ -279,8 +279,8 @@ TEST(InMemFishStore, Ingest_Concurrent) { TEST(InMemFishStore, FullScan) { store_t store{ 1LL << 24, 1LL << 30, "" }; store.StartSession(); - auto id_proj = store.MakeProjection("id"); - auto gender_proj = store.MakeProjection("gender"); + auto id_proj = store.MakeProjection("/id"); + auto gender_proj = store.MakeProjection("/gender"); std::vector actions; actions.push_back({ REGISTER_GENERAL_PSF, id_proj }); actions.push_back({ REGISTER_GENERAL_PSF, gender_proj }); @@ -319,12 +319,12 @@ TEST(InMemFishStore, FullScan) { ASSERT_EQ(result, Status::Ok); }; - JsonFullScanContext context1{ {"id"}, fishstore::core::projection, "1234", 1 }; + JsonFullScanContext context1{ {"/id"}, fishstore::core::projection, "1234", 1 }; auto res = store.FullScan(context1, callback, 0); ASSERT_EQ(res, Status::Ok); store.CompletePending(); - JsonFullScanContext context2{ {"gender"}, fishstore::core::projection, "male", n_records / 2 }; + JsonFullScanContext context2{ {"/gender"}, fishstore::core::projection, "male", n_records / 2 }; res = store.FullScan(context2, callback, 0); ASSERT_EQ(res, Status::Ok); store.CompletePending(); diff --git a/test/ingest_test.h b/test/ingest_test.h index 85cd497..55d8a3d 100644 --- a/test/ingest_test.h +++ b/test/ingest_test.h @@ -161,8 +161,8 @@ TEST(CLASS, Ingest_Serial) { std::experimental::filesystem::create_directories("test"); store_t store{ 8192, 201326592, "test" }; store.StartSession(); - auto id_proj = store.MakeProjection("id"); - auto gender_proj = store.MakeProjection("gender"); + auto id_proj = store.MakeProjection("/id"); + auto gender_proj = store.MakeProjection("/gender"); std::vector actions; actions.push_back({ REGISTER_GENERAL_PSF, id_proj }); actions.push_back({ REGISTER_GENERAL_PSF, gender_proj }); @@ -217,8 +217,8 @@ TEST(CLASS, Ingest_Concurrent) { { store_t store{ 8192, 201326592, "test" }; store.StartSession(); - auto id_proj = store.MakeProjection("id"); - auto gender_proj = store.MakeProjection("gender"); + auto id_proj = store.MakeProjection("/id"); + auto gender_proj = store.MakeProjection("/gender"); std::vector actions; actions.push_back({ REGISTER_GENERAL_PSF, id_proj }); actions.push_back({ REGISTER_GENERAL_PSF, gender_proj }); @@ -286,8 +286,8 @@ TEST(CLASS, FullScan) { std::experimental::filesystem::create_directories("test"); store_t store{ 8192, 201326592, "test" }; store.StartSession(); - auto id_proj = store.MakeProjection("id"); - auto gender_proj = store.MakeProjection("gender"); + auto id_proj = store.MakeProjection("/id"); + auto gender_proj = store.MakeProjection("/gender"); std::vector actions; actions.push_back({ REGISTER_GENERAL_PSF, id_proj }); actions.push_back({ REGISTER_GENERAL_PSF, gender_proj }); @@ -335,12 +335,12 @@ TEST(CLASS, FullScan) { ASSERT_EQ(result, Status::Ok); }; - JsonFullScanContext context1{ {"id"}, fishstore::core::projection, "1234", 1 }; + JsonFullScanContext context1{ {"/id"}, fishstore::core::projection, "1234", 1 }; auto res = store.FullScan(context1, callback, 0); ASSERT_EQ(res, Status::Pending); store.CompletePending(true); - JsonFullScanContext context2{ {"gender"}, fishstore::core::projection, "male", n_records / 2 }; + JsonFullScanContext context2{ {"/gender"}, fishstore::core::projection, "male", n_records / 2 }; res = store.FullScan(context2, callback, 0); ASSERT_EQ(res, Status::Pending); store.CompletePending(true); diff --git a/test/register_test.cc b/test/register_test.cc index 2e9b92b..3e8af94 100644 --- a/test/register_test.cc +++ b/test/register_test.cc @@ -175,7 +175,7 @@ TEST(Registration, Register_Concurrent) { std::experimental::filesystem::create_directories("test"); store_t store{ 8192, 201326592, "test" }; store.StartSession(); - auto school_id_proj = store.MakeProjection("school.id"); + auto school_id_proj = store.MakeProjection("/school/id"); std::vector actions; actions.push_back({ REGISTER_GENERAL_PSF, school_id_proj }); @@ -220,7 +220,7 @@ TEST(Registration, Register_Concurrent) { auto res = store.Scan(context1, callback, 0, safe_register_address); store.CompletePending(true); - JsonFullScanContext context2{ {"school.id"}, fishstore::core::projection, "1" }; + JsonFullScanContext context2{ {"/school/id"}, fishstore::core::projection, "1" }; res = store.FullScan(context2, callback, 0, safe_register_address); store.CompletePending(true); @@ -243,7 +243,7 @@ TEST(Registration, Deregister_Concurrent) { std::experimental::filesystem::create_directories("test"); store_t store{ 8192, 201326592, "test" }; store.StartSession(); - auto school_id_proj = store.MakeProjection("school.id"); + auto school_id_proj = store.MakeProjection("/school/id"); std::vector actions; actions.push_back({ REGISTER_GENERAL_PSF, school_id_proj }); uint64_t safe_register_address, safe_unregister_address; @@ -295,7 +295,7 @@ TEST(Registration, Deregister_Concurrent) { auto res = store.Scan(context1, callback, 0, 0, safe_unregister_address); store.CompletePending(true); - JsonFullScanContext context2{ {"school.id"}, fishstore::core::projection, "1" }; + JsonFullScanContext context2{ {"/school/id"}, fishstore::core::projection, "1" }; res = store.FullScan(context2, callback, 0, 0, safe_unregister_address); store.CompletePending(true); diff --git a/third_party/hopscotch-map b/third_party/hopscotch-map index 41775f3..7ef9cc4 160000 --- a/third_party/hopscotch-map +++ b/third_party/hopscotch-map @@ -1 +1 @@ -Subproject commit 41775f37f2d2a6962d038378c22a7f3e878180c2 +Subproject commit 7ef9cc4aca326cbe725553ca330d74076f936ff2 diff --git a/third_party/simdjson b/third_party/simdjson index 3db8c5a..ee66fb1 160000 --- a/third_party/simdjson +++ b/third_party/simdjson @@ -1 +1 @@ -Subproject commit 3db8c5a0eb3c02b56fc8858731a3ae17cd2ea2ec +Subproject commit ee66fb1c602e17563606c6f6eecc225dac5455cc