Adapt simdjson 0.2.0

CMakeLists.txt

@@ -10,7 +10,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 if (MSVC)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /Gm- /W3 /WX /EHsc /GS /fp:precise /permissive- /Zc:wchar_t /Zc:forScope /Zc:inline /Gd /TP")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /D_CRT_SECURE_NO_WARNINGS /FC")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /D_CRT_SECURE_NO_WARNINGS /FC /wd4244 /wd4267")
 
     set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /MDd")
     set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /O2 /Oi /Gy- /MD")

examples/checkpoint_recovery-dir/checkpoint_recovery.cc

@@ -219,12 +219,12 @@ int main(int argc, char* argv[]) {
     store.StartSession();
 
     auto lib_id = store.LoadPSFLibrary(argv[2]);
-    auto id_proj = store.MakeProjection("id");
-    auto actor_id_proj = store.MakeProjection("actor.id");
-    auto repo_id_proj = store.MakeProjection("repo.id");
-    auto type_proj = store.MakeProjection("type");
-    auto predicate1_id = store.MakeInlinePSF({ "type", "payload.action" }, lib_id, "opened_issue");
-    auto predicate2_id = store.MakeInlinePSF({ "type", "payload.pull_request.head.repo.language" }, lib_id, "cpp_pr");
+    auto id_proj = store.MakeProjection("/id");
+    auto actor_id_proj = store.MakeProjection("/actor/id");
+    auto repo_id_proj = store.MakeProjection("/repo/id");
+    auto type_proj = store.MakeProjection("/type");
+    auto predicate1_id = store.MakeInlinePSF({ "/type", "/payload/action" }, lib_id, "opened_issue");
+    auto predicate2_id = store.MakeInlinePSF({ "/type", "/payload/pull_request/head/repo/language" }, lib_id, "cpp_pr");
 
     std::vector<ParserAction> parser_actions;
     parser_actions.push_back({ REGISTER_GENERAL_PSF, id_proj });

examples/github_query-dir/github_query.cc

@@ -206,14 +206,14 @@ int main(int argc, char* argv[]) {
   store.StartSession();
 
   auto lib_id = store.LoadPSFLibrary(argv[2]);
-  auto id_proj = store.MakeProjection("id");
-  auto actor_id_proj = store.MakeInlinePSF({ "actor.id" }, lib_id, "int_projection");
-  auto repo_id_proj = store.MakeInlinePSF({ "repo.id" }, lib_id, "int_projection");
-  auto type_proj = store.MakeProjection("type");
+  auto id_proj = store.MakeProjection("/id");
+  auto actor_id_proj = store.MakeInlinePSF({ "/actor/id" }, lib_id, "int_projection");
+  auto repo_id_proj = store.MakeInlinePSF({ "/repo/id" }, lib_id, "int_projection");
+  auto type_proj = store.MakeProjection("/type");
   auto predicate1_id =
-    store.MakeInlinePSF({ "type", "payload.action" }, lib_id, "opened_issue");
+    store.MakeInlinePSF({ "/type", "/payload/action" }, lib_id, "opened_issue");
   auto predicate2_id =
-    store.MakeInlinePSF({ "type", "payload.pull_request.head.repo.language" }, lib_id, "cpp_pr");
+    store.MakeInlinePSF({ "/type", "/payload/pull_request/head.repo/language" }, lib_id, "cpp_pr");
 
   std::vector<ParserAction> parser_actions;
   parser_actions.push_back({ REGISTER_GENERAL_PSF, id_proj });

examples/twitter_query-dir/twitter_query.cc

@@ -206,15 +206,15 @@ int main(int argc, char* argv[]) {
   store.StartSession();
 
   auto lib_id = store.LoadPSFLibrary(argv[2]);
-  auto id_proj = store.MakeProjection("id");
-  auto user_id_proj = store.MakeProjection("user.id");
-  auto reply_status_id_proj = store.MakeProjection("in_reply_to_status_id");
-  auto reply_user_id_proj = store.MakeProjection("in_reply_to_user_id");
-  auto lang_proj = store.MakeProjection("lang");
+  auto id_proj = store.MakeProjection("/id");
+  auto user_id_proj = store.MakeProjection("/user/id");
+  auto reply_status_id_proj = store.MakeProjection("/in_reply_to_status_id");
+  auto reply_user_id_proj = store.MakeProjection("/in_reply_to_user_id");
+  auto lang_proj = store.MakeProjection("/lang");
   auto predicate1_id =
-    store.MakeInlinePSF({ "user.lang", "user.followers_count" }, lib_id, "jp_high_follower");
+    store.MakeInlinePSF({ "/user/lang", "/user/followers_count" }, lib_id, "jp_high_follower");
   auto predicate2_id =
-    store.MakeInlinePSF({ "in_reply_to_screen_name", "possibly_sensitive" }, lib_id, "sensitive_reply_to_trump");
+    store.MakeInlinePSF({ "/in_reply_to_screen_name", "/possibly_sensitive" }, lib_id, "sensitive_reply_to_trump");
 
   std::vector<ParserAction> parser_actions;
   parser_actions.push_back({ REGISTER_GENERAL_PSF, id_proj });

src/adapters/simdjson_adapter.h

@@ -8,27 +8,21 @@
 #include <cassert>
 #include <vector>
 
+#ifdef _MSC_VER
+#define NOMINMAX
+#endif
 #include <simdjson/parsedjson.h>
 #include <simdjson/jsonparser.h>
 #include "adapters/common_utils.h"
 
+using namespace simdjson;
+
 namespace fishstore {
 namespace adapter {
 
-struct TreeNode {
-  TreeNode(): field_id(-1) {}
-  TreeNode(int64_t id): field_id(id) {}
-  ~TreeNode() {
-    for (auto x: children) delete x.second;
-  }
-
-  int64_t field_id = -1;
-  std::unordered_map<std::string, TreeNode*> children;
-};
-
 class SIMDJsonField {
 public:
-  SIMDJsonField(int64_t id_, const ParsedJson::iterator& it_)
+  SIMDJsonField(int64_t id_, const ParsedJson::Iterator& it_)
     : field_id(id_), iter(it_) {}
 
   inline int64_t FieldId() const {
@@ -84,7 +78,7 @@ class SIMDJsonField {
 
 private:
   int64_t field_id;
-  ParsedJson::iterator iter;
+  ParsedJson::Iterator iter;
 };
 
 class SIMDJsonRecord {
@@ -113,46 +107,18 @@ class SIMDJsonRecord {
 
 class SIMDJsonParser {
 public:
-  SIMDJsonParser(const std::vector<std::string>& field_names, const size_t alloc_bytes = 1LL << 25) {
-    root = new TreeNode{};
-    for (auto field_id = 0; field_id < field_names.size(); ++field_id) {
-      auto& field_name = field_names[field_id];
-      auto current_tree_node = root;
-      size_t start_pos = 0, pos = 0;
-      while (std::string::npos != (pos = field_name.find('.', start_pos))) {
-        auto slice = field_name.substr(start_pos, pos - start_pos);
-        auto iter = current_tree_node->children.find(slice);
-        if (iter != current_tree_node->children.end()) {
-          current_tree_node = iter->second;
-        }
-        else {
-          auto new_node = current_tree_node->children.emplace(std::make_pair(slice, new TreeNode{}));
-          current_tree_node = new_node.first->second;
-        }
-        start_pos = pos + 1;
-      }
-
-      auto slice = field_name.substr(start_pos);
-      auto iter = current_tree_node->children.find(slice);
-      if (iter != current_tree_node->children.end()) {
-        printf("Skipping field %s as seen before...\n", field_name.c_str());
-      }
-      else current_tree_node->children.emplace(std::make_pair(slice, new TreeNode{ field_id }));
-    }
-    auto success = pj.allocateCapacity(alloc_bytes);
+  SIMDJsonParser(const std::vector<std::string>& field_names, const size_t alloc_bytes = 1LL << 25)
+  : fields(field_names) {
+    auto success = pj.allocate_capacity(alloc_bytes);
     assert(success);
     has_next = false;
   }
 
-  ~SIMDJsonParser() {
-    delete root;
-  }
-
   inline void Load(const char* buffer, size_t length) {
     record.original = StringRef(buffer, length);
     record.fields.clear();
     auto ok = json_parse(buffer, length, pj);
-    if (ok != 0 || !pj.isValid()) {
+    if (ok != 0 || !pj.is_valid()) {
       printf("Parsing failed...\n");
       has_next = false;
     } else {
@@ -165,35 +131,18 @@ class SIMDJsonParser {
   }
 
   inline const SIMDJsonRecord& NextRecord() {
-    ParsedJson::iterator it(pj);
-    Traverse(it, root, record.fields);
+    ParsedJson::Iterator it(pj);
+    for (auto field_id = 0; field_id < fields.size(); ++field_id) {
+      if (it.move_to(fields[field_id])) {
+        record.fields.emplace_back(SIMDJsonField{field_id, it});
+      }
+    }
     has_next = false;
     return record;
   }
 
 private:
-  void Traverse(ParsedJson::iterator& it, const TreeNode* current_tree_node,
-    std::vector<SIMDJsonField>& parsed_fields) {
-
-    if (current_tree_node->field_id != -1) {
-      parsed_fields.emplace_back(SIMDJsonField(current_tree_node->field_id, it));
-    }
-    if (!current_tree_node->children.empty() && it.is_object()) {
-      if (it.down()) {
-        do {
-          std::string slice(it.get_string());
-          auto slice_iter = current_tree_node->children.find(slice);
-          it.next();
-          if (slice_iter != current_tree_node->children.end()) {
-            Traverse(it, slice_iter->second, parsed_fields);
-          }
-        } while (it.next());
-        it.up();
-      }
-    }
-  }
-
-  TreeNode* root;
+  std::vector<std::string> fields;
   ParsedJson pj;
   SIMDJsonRecord record;
   bool has_next;

src/core/fishstore.h

@@ -15,6 +15,7 @@
 #include <unordered_map>
 #include <experimental/filesystem>
 #include <fstream>
+#include <limits>
 
 #include "device/file_system_disk.h"
 #include "tsl/hopscotch_map.h"
@@ -1403,7 +1404,7 @@ inline OperationStatus FishStore<D, A>::InternalScan(C& pending_context,
   uint64_t latest_record_version = 0;
 
   const KeyPointer* kpt = reinterpret_cast<const KeyPointer*>(hlog.Get(address));
-  while(address >= end_addr && address >= head_address) {
+  while(address > end_addr && address >= head_address) {
     address = kpt->prev_address;
     kpt = reinterpret_cast<const KeyPointer*>(hlog.Get(address));
   }
@@ -1482,7 +1483,7 @@ inline OperationStatus FishStore<D, A>::InternalFullScan(C& pending_context,
     if(current_page < hlog.head_address.load().page()) break;
 
     // Scan the page, record by record to the end of the page or current tail.
-    while(address <= to_address) {
+    while(address < to_address) {
       record_t* record = reinterpret_cast<record_t*>(hlog.Get(address));
       if(record->header.IsNull()) {
         address += sizeof(RecordInfo);

test/checkpoint_test.h

@@ -165,8 +165,8 @@ TEST(CLASS, Checkpoint_Concurrent) {
   {
     store_t store{ 8192, 201326592, "test" };
     store.StartSession();
-    auto id_proj = store.MakeProjection("id");
-    auto gender_proj = store.MakeProjection("gender");
+    auto id_proj = store.MakeProjection("/id");
+    auto gender_proj = store.MakeProjection("/gender");
     std::vector<ParserAction> actions;
     actions.push_back({ REGISTER_GENERAL_PSF, id_proj });
     actions.push_back({ REGISTER_GENERAL_PSF, gender_proj });

test/in_memory_test.cc

@@ -160,8 +160,8 @@ class JsonFullScanContext : public IAsyncContext {
 TEST(InMemFishStore, Ingest_Serial) {
   store_t store{ 1LL << 24, 1LL << 30, "" };
   store.StartSession();
-  auto id_proj = store.MakeProjection("id");
-  auto gender_proj = store.MakeProjection("gender");
+  auto id_proj = store.MakeProjection("/id");
+  auto gender_proj = store.MakeProjection("/gender");
   std::vector<ParserAction> actions;
   actions.push_back({ REGISTER_GENERAL_PSF, id_proj });
   actions.push_back({ REGISTER_GENERAL_PSF, gender_proj });
@@ -213,8 +213,8 @@ TEST(InMemFishStore, Ingest_Serial) {
 TEST(InMemFishStore, Ingest_Concurrent) {
   store_t store{ 1LL << 24, 1LL << 30, "" };
   store.StartSession();
-  auto id_proj = store.MakeProjection("id");
-  auto gender_proj = store.MakeProjection("gender");
+  auto id_proj = store.MakeProjection("/id");
+  auto gender_proj = store.MakeProjection("/gender");
   std::vector<ParserAction> actions;
   actions.push_back({ REGISTER_GENERAL_PSF, id_proj });
   actions.push_back({ REGISTER_GENERAL_PSF, gender_proj });
@@ -279,8 +279,8 @@ TEST(InMemFishStore, Ingest_Concurrent) {
 TEST(InMemFishStore, FullScan) {
   store_t store{ 1LL << 24, 1LL << 30, "" };
   store.StartSession();
-  auto id_proj = store.MakeProjection("id");
-  auto gender_proj = store.MakeProjection("gender");
+  auto id_proj = store.MakeProjection("/id");
+  auto gender_proj = store.MakeProjection("/gender");
   std::vector<ParserAction> actions;
   actions.push_back({ REGISTER_GENERAL_PSF, id_proj });
   actions.push_back({ REGISTER_GENERAL_PSF, gender_proj });
@@ -319,12 +319,12 @@ TEST(InMemFishStore, FullScan) {
     ASSERT_EQ(result, Status::Ok);
   };
 
-  JsonFullScanContext context1{ {"id"}, fishstore::core::projection<adapter_t>, "1234", 1 };
+  JsonFullScanContext context1{ {"/id"}, fishstore::core::projection<adapter_t>, "1234", 1 };
   auto res = store.FullScan(context1, callback, 0);
   ASSERT_EQ(res, Status::Ok);
   store.CompletePending();
 
-  JsonFullScanContext context2{ {"gender"}, fishstore::core::projection<adapter_t>, "male", n_records / 2 };
+  JsonFullScanContext context2{ {"/gender"}, fishstore::core::projection<adapter_t>, "male", n_records / 2 };
   res = store.FullScan(context2, callback, 0);
   ASSERT_EQ(res, Status::Ok);
   store.CompletePending();

test/ingest_test.h

@@ -161,8 +161,8 @@ TEST(CLASS, Ingest_Serial) {
   std::experimental::filesystem::create_directories("test");
   store_t store{ 8192, 201326592, "test" };
   store.StartSession();
-  auto id_proj = store.MakeProjection("id");
-  auto gender_proj = store.MakeProjection("gender");
+  auto id_proj = store.MakeProjection("/id");
+  auto gender_proj = store.MakeProjection("/gender");
   std::vector<ParserAction> actions;
   actions.push_back({ REGISTER_GENERAL_PSF, id_proj });
   actions.push_back({ REGISTER_GENERAL_PSF, gender_proj });
@@ -217,8 +217,8 @@ TEST(CLASS, Ingest_Concurrent) {
   {
     store_t store{ 8192, 201326592, "test" };
     store.StartSession();
-    auto id_proj = store.MakeProjection("id");
-    auto gender_proj = store.MakeProjection("gender");
+    auto id_proj = store.MakeProjection("/id");
+    auto gender_proj = store.MakeProjection("/gender");
     std::vector<ParserAction> actions;
     actions.push_back({ REGISTER_GENERAL_PSF, id_proj });
     actions.push_back({ REGISTER_GENERAL_PSF, gender_proj });
@@ -286,8 +286,8 @@ TEST(CLASS, FullScan) {
   std::experimental::filesystem::create_directories("test");
   store_t store{ 8192, 201326592, "test" };
   store.StartSession();
-  auto id_proj = store.MakeProjection("id");
-  auto gender_proj = store.MakeProjection("gender");
+  auto id_proj = store.MakeProjection("/id");
+  auto gender_proj = store.MakeProjection("/gender");
   std::vector<ParserAction> actions;
   actions.push_back({ REGISTER_GENERAL_PSF, id_proj });
   actions.push_back({ REGISTER_GENERAL_PSF, gender_proj });
@@ -335,12 +335,12 @@ TEST(CLASS, FullScan) {
     ASSERT_EQ(result, Status::Ok);
   };
 
-  JsonFullScanContext context1{ {"id"}, fishstore::core::projection<adapter_t>, "1234", 1 };
+  JsonFullScanContext context1{ {"/id"}, fishstore::core::projection<adapter_t>, "1234", 1 };
   auto res = store.FullScan(context1, callback, 0);
   ASSERT_EQ(res, Status::Pending);
   store.CompletePending(true);
 
-  JsonFullScanContext context2{ {"gender"}, fishstore::core::projection<adapter_t>, "male", n_records / 2 };
+  JsonFullScanContext context2{ {"/gender"}, fishstore::core::projection<adapter_t>, "male", n_records / 2 };
   res = store.FullScan(context2, callback, 0);
   ASSERT_EQ(res, Status::Pending);
   store.CompletePending(true);

test/register_test.cc

@@ -175,7 +175,7 @@ TEST(Registration, Register_Concurrent) {
   std::experimental::filesystem::create_directories("test");
   store_t store{ 8192, 201326592, "test" };
   store.StartSession();
-  auto school_id_proj = store.MakeProjection("school.id");
+  auto school_id_proj = store.MakeProjection("/school/id");
   std::vector<ParserAction> actions;
   actions.push_back({ REGISTER_GENERAL_PSF, school_id_proj });
 
@@ -220,7 +220,7 @@ TEST(Registration, Register_Concurrent) {
   auto res = store.Scan(context1, callback, 0, safe_register_address);
   store.CompletePending(true);
 
-  JsonFullScanContext context2{ {"school.id"}, fishstore::core::projection<adapter_t>, "1" };
+  JsonFullScanContext context2{ {"/school/id"}, fishstore::core::projection<adapter_t>, "1" };
   res = store.FullScan(context2, callback, 0, safe_register_address);
   store.CompletePending(true);
 
@@ -243,7 +243,7 @@ TEST(Registration, Deregister_Concurrent) {
   std::experimental::filesystem::create_directories("test");
   store_t store{ 8192, 201326592, "test" };
   store.StartSession();
-  auto school_id_proj = store.MakeProjection("school.id");
+  auto school_id_proj = store.MakeProjection("/school/id");
   std::vector<ParserAction> actions;
   actions.push_back({ REGISTER_GENERAL_PSF, school_id_proj });
   uint64_t safe_register_address, safe_unregister_address;
@@ -295,7 +295,7 @@ TEST(Registration, Deregister_Concurrent) {
   auto res = store.Scan(context1, callback, 0, 0, safe_unregister_address);
   store.CompletePending(true);
 
-  JsonFullScanContext context2{ {"school.id"}, fishstore::core::projection<adapter_t>, "1" };
+  JsonFullScanContext context2{ {"/school/id"}, fishstore::core::projection<adapter_t>, "1" };
   res = store.FullScan(context2, callback, 0, 0, safe_unregister_address);
   store.CompletePending(true);