Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions examples/hello_world/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

include(FetchContent)
FetchContent_Declare(highway GIT_REPOSITORY https://github.com/google/highway.git GIT_TAG 9414b48aeec251b69e6cadbfa42bebb5ddae1c34)
FetchContent_Declare(highway GIT_REPOSITORY https://github.com/google/highway.git GIT_TAG bfc36a6e633af94e63ac4b91c687bf0354cb24e0)
FetchContent_MakeAvailable(highway)
FetchContent_Declare(sentencepiece GIT_REPOSITORY https://github.com/google/sentencepiece GIT_TAG 53de76561cfc149d3c01037f0595669ad32a5e7c)
FetchContent_Declare(sentencepiece GIT_REPOSITORY https://github.com/google/sentencepiece GIT_TAG 9045b2f60fa2b323dfac0eaef8fc17565036f9f9)
FetchContent_MakeAvailable(sentencepiece)


Expand All @@ -34,7 +34,7 @@ if (BUILD_MODE STREQUAL "local")
# Relative path to gemma.cpp from examples/hello_world/build/
FetchContent_Declare(gemma SOURCE_DIR ../../..)
else()
FetchContent_Declare(gemma GIT_REPOSITORY https://github.com/google/gemma.cpp.git GIT_TAG a9aa63fd2ea6b786ed0706d619588bfe2d43370e)
FetchContent_Declare(gemma GIT_REPOSITORY https://github.com/google/gemma.cpp.git GIT_TAG 4a924f179448dc83e46a2af9520c61b4ef56174c)
endif()
FetchContent_MakeAvailable(gemma)

Expand Down
121 changes: 63 additions & 58 deletions examples/hello_world/run.cc
Original file line number Diff line number Diff line change
Expand Up @@ -30,69 +30,74 @@
#include "util/threading_context.h"
#include "hwy/base.h"

int main(int argc, char** argv) {
gcpp::LoaderArgs loader(argc, argv);
gcpp::ThreadingArgs threading(argc, argv);
gcpp::InferenceArgs inference(argc, argv);
if (gcpp::HasHelp(argc, argv)) {
loader.Help();
return 0;
}

// Demonstrate constrained decoding by never outputting certain tokens.
std::set<int> reject_tokens;
for (int arg = 0; arg < argc; ++arg) {
// Find a --reject flag and consume everything after it.
if (strcmp(argv[arg], "--reject") == 0) {
while (++arg < argc) {
reject_tokens.insert(atoi(argv[arg])); // NOLINT
}
int main(int argc, char **argv) {
gcpp::LoaderArgs loader(argc, argv);
gcpp::InferenceArgs inference(argc, argv);
gcpp::AppArgs app(argc, argv);
if (gcpp::HasHelp(argc, argv)) {
loader.Help();
return 0;
} else if (const char *error = loader.Validate()) {
loader.Help();
HWY_ABORT("\nInvalid args: %s", error);
}
}

// Instantiate model and KV Cache
gcpp::ThreadingContext ctx(gcpp::UpdateArgs(threading, inference));
gcpp::MatMulEnv env(ctx);
gcpp::Gemma gemma(loader, inference, ctx);
gcpp::KVCache kv_cache(gemma.Config(), inference, ctx.allocator);
size_t generated = 0;
// Demonstrate constrained decoding by never outputting certain tokens.
std::set<int> reject_tokens;
for (int arg = 0; arg < argc; ++arg) {
// Find a --reject flag and consume everything after it.
if (strcmp(argv[arg], "--reject") == 0) {
while (++arg < argc) reject_tokens.insert(atoi(argv[arg]));
}
}

// Initialize random number generator
std::mt19937 gen;
std::random_device rd; // NOLINT
gen.seed(rd());
// Instantiate model and KV Cache
gcpp::BoundedTopology topology(gcpp::CreateTopology(app));
gcpp::NestedPools pools = gcpp::CreatePools(topology, app);
gcpp::MatMulEnv env(topology, pools);
gcpp::Gemma model = gcpp::CreateGemma(loader, env);
gcpp::KVCache kv_cache =
gcpp::KVCache::Create(model.GetModelConfig(),
inference.prefill_tbatch_size);
size_t generated = 0;

// Tokenize instructions.
std::string prompt = "Write a greeting to the world.";
const std::vector<int> tokens =
gcpp::WrapAndTokenize(gemma.Tokenizer(), gemma.ChatTemplate(),
gemma.Config().wrapping, generated, prompt);
const size_t prompt_size = tokens.size();
// Initialize random number generator
std::mt19937 gen;
std::random_device rd;
gen.seed(rd());

// This callback function gets invoked every time a token is generated
auto stream_token = [&generated, &prompt_size, &gemma](int token, float) {
++generated;
if (generated < prompt_size) {
// print feedback
} else if (!gemma.Config().IsEOS(token)) {
std::string token_text;
HWY_ASSERT(gemma.Tokenizer().Decode({token}, &token_text));
std::cout << token_text << std::flush;
}
return true;
};
// Tokenize instructions.
std::string prompt = "Write a greeting to the world.";
const std::vector<int> tokens = gcpp::WrapAndTokenize(
model.Tokenizer(), loader.Info(), generated, prompt);
const size_t prompt_size = tokens.size();

// This callback function gets invoked every time a token is generated
auto stream_token = [&generated, &prompt_size, &model](int token, float) {
++generated;
if (generated < prompt_size) {
// print feedback
} else if (!model.GetModelConfig().IsEOS(token)) {
std::string token_text;
HWY_ASSERT(model.Tokenizer().Decode({token}, &token_text));
std::cout << token_text << std::flush;
}
return true;
};

gcpp::TimingInfo timing_info;
gcpp::RuntimeConfig runtime_config = {
.max_generated_tokens = 1024,
.temperature = 1.0,
.gen = &gen,
.verbosity = 0,
.stream_token = stream_token,
.accept_token =
[&](int token, float /* prob */) {
return !reject_tokens.contains(token);
},
};
gemma.Generate(runtime_config, tokens, 0, kv_cache, env, timing_info);
gcpp::TimingInfo timing_info;
gcpp::RuntimeConfig runtime_config = {
.max_generated_tokens = 1024,
.temperature = 1.0,
.gen = &gen,
.verbosity = 0,
.stream_token = stream_token,
.accept_token =
std::function<bool(int, float)>(
[&](int token, float /* prob */) {
return reject_tokens.find(token) == reject_tokens.end();
}),
};
model.Generate(runtime_config, tokens, 0, kv_cache, timing_info);
}