diff --git a/examples/hello_world/CMakeLists.txt b/examples/hello_world/CMakeLists.txt index be9a108e..2f5d648d 100644 --- a/examples/hello_world/CMakeLists.txt +++ b/examples/hello_world/CMakeLists.txt @@ -18,9 +18,9 @@ set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) include(FetchContent) -FetchContent_Declare(highway GIT_REPOSITORY https://github.com/google/highway.git GIT_TAG 9414b48aeec251b69e6cadbfa42bebb5ddae1c34) +FetchContent_Declare(highway GIT_REPOSITORY https://github.com/google/highway.git GIT_TAG bfc36a6e633af94e63ac4b91c687bf0354cb24e0) FetchContent_MakeAvailable(highway) -FetchContent_Declare(sentencepiece GIT_REPOSITORY https://github.com/google/sentencepiece GIT_TAG 53de76561cfc149d3c01037f0595669ad32a5e7c) +FetchContent_Declare(sentencepiece GIT_REPOSITORY https://github.com/google/sentencepiece GIT_TAG 9045b2f60fa2b323dfac0eaef8fc17565036f9f9) FetchContent_MakeAvailable(sentencepiece) @@ -34,7 +34,7 @@ if (BUILD_MODE STREQUAL "local") # Relative path to gemma.cpp from examples/hello_world/build/ FetchContent_Declare(gemma SOURCE_DIR ../../..) else() - FetchContent_Declare(gemma GIT_REPOSITORY https://github.com/google/gemma.cpp.git GIT_TAG a9aa63fd2ea6b786ed0706d619588bfe2d43370e) + FetchContent_Declare(gemma GIT_REPOSITORY https://github.com/google/gemma.cpp.git GIT_TAG 4a924f179448dc83e46a2af9520c61b4ef56174c) endif() FetchContent_MakeAvailable(gemma) diff --git a/examples/hello_world/run.cc b/examples/hello_world/run.cc index 9f124077..96c56bfd 100644 --- a/examples/hello_world/run.cc +++ b/examples/hello_world/run.cc @@ -30,69 +30,74 @@ #include "util/threading_context.h" #include "hwy/base.h" -int main(int argc, char** argv) { - gcpp::LoaderArgs loader(argc, argv); - gcpp::ThreadingArgs threading(argc, argv); - gcpp::InferenceArgs inference(argc, argv); - if (gcpp::HasHelp(argc, argv)) { - loader.Help(); - return 0; - } - // Demonstrate constrained decoding by never outputting certain tokens. - std::set reject_tokens; - for (int arg = 0; arg < argc; ++arg) { - // Find a --reject flag and consume everything after it. - if (strcmp(argv[arg], "--reject") == 0) { - while (++arg < argc) { - reject_tokens.insert(atoi(argv[arg])); // NOLINT - } +int main(int argc, char **argv) { + gcpp::LoaderArgs loader(argc, argv); + gcpp::InferenceArgs inference(argc, argv); + gcpp::AppArgs app(argc, argv); + if (gcpp::HasHelp(argc, argv)) { + loader.Help(); + return 0; + } else if (const char *error = loader.Validate()) { + loader.Help(); + HWY_ABORT("\nInvalid args: %s", error); } - } - // Instantiate model and KV Cache - gcpp::ThreadingContext ctx(gcpp::UpdateArgs(threading, inference)); - gcpp::MatMulEnv env(ctx); - gcpp::Gemma gemma(loader, inference, ctx); - gcpp::KVCache kv_cache(gemma.Config(), inference, ctx.allocator); - size_t generated = 0; + // Demonstrate constrained decoding by never outputting certain tokens. + std::set reject_tokens; + for (int arg = 0; arg < argc; ++arg) { + // Find a --reject flag and consume everything after it. + if (strcmp(argv[arg], "--reject") == 0) { + while (++arg < argc) reject_tokens.insert(atoi(argv[arg])); + } + } - // Initialize random number generator - std::mt19937 gen; - std::random_device rd; // NOLINT - gen.seed(rd()); + // Instantiate model and KV Cache + gcpp::BoundedTopology topology(gcpp::CreateTopology(app)); + gcpp::NestedPools pools = gcpp::CreatePools(topology, app); + gcpp::MatMulEnv env(topology, pools); + gcpp::Gemma model = gcpp::CreateGemma(loader, env); + gcpp::KVCache kv_cache = + gcpp::KVCache::Create(model.GetModelConfig(), + inference.prefill_tbatch_size); + size_t generated = 0; - // Tokenize instructions. - std::string prompt = "Write a greeting to the world."; - const std::vector tokens = - gcpp::WrapAndTokenize(gemma.Tokenizer(), gemma.ChatTemplate(), - gemma.Config().wrapping, generated, prompt); - const size_t prompt_size = tokens.size(); + // Initialize random number generator + std::mt19937 gen; + std::random_device rd; + gen.seed(rd()); - // This callback function gets invoked every time a token is generated - auto stream_token = [&generated, &prompt_size, &gemma](int token, float) { - ++generated; - if (generated < prompt_size) { - // print feedback - } else if (!gemma.Config().IsEOS(token)) { - std::string token_text; - HWY_ASSERT(gemma.Tokenizer().Decode({token}, &token_text)); - std::cout << token_text << std::flush; - } - return true; - }; + // Tokenize instructions. + std::string prompt = "Write a greeting to the world."; + const std::vector tokens = gcpp::WrapAndTokenize( + model.Tokenizer(), loader.Info(), generated, prompt); + const size_t prompt_size = tokens.size(); + + // This callback function gets invoked every time a token is generated + auto stream_token = [&generated, &prompt_size, &model](int token, float) { + ++generated; + if (generated < prompt_size) { + // print feedback + } else if (!model.GetModelConfig().IsEOS(token)) { + std::string token_text; + HWY_ASSERT(model.Tokenizer().Decode({token}, &token_text)); + std::cout << token_text << std::flush; + } + return true; + }; - gcpp::TimingInfo timing_info; - gcpp::RuntimeConfig runtime_config = { - .max_generated_tokens = 1024, - .temperature = 1.0, - .gen = &gen, - .verbosity = 0, - .stream_token = stream_token, - .accept_token = - [&](int token, float /* prob */) { - return !reject_tokens.contains(token); - }, - }; - gemma.Generate(runtime_config, tokens, 0, kv_cache, env, timing_info); + gcpp::TimingInfo timing_info; + gcpp::RuntimeConfig runtime_config = { + .max_generated_tokens = 1024, + .temperature = 1.0, + .gen = &gen, + .verbosity = 0, + .stream_token = stream_token, + .accept_token = + std::function( + [&](int token, float /* prob */) { + return reject_tokens.find(token) == reject_tokens.end(); + }), + }; + model.Generate(runtime_config, tokens, 0, kv_cache, timing_info); }