diff --git a/BUILD.bazel b/BUILD.bazel index ba94242f..2e731400 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1035,6 +1035,23 @@ cc_library( ], ) +cc_test( + name = "embeddings_test", + srcs = ["gemma/embeddings_test.cc"], + deps = [ + ":configs", + ":gemma_args", + ":gemma_lib", + ":mat", + ":matmul", + ":threading_context", + ":weights", + "@googletest//:gtest_main", # buildcleaner: keep + "//compression:types", + "@highway//:hwy", + ], +) + cc_test( name = "gemma_test", srcs = [ diff --git a/CMakeLists.txt b/CMakeLists.txt index a3d2287c..70479da8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -376,6 +376,7 @@ set(GEMMA_TEST_FILES compression/q4_0_test.cc compression/sfp_test.cc deepseek/deepseek_test.cc + gemma/embeddings_test.cc gemma/gemma_args_test.cc gemma/tensor_info_test.cc gemma/weights_test.cc diff --git a/gemma/activations.h b/gemma/activations.h index 86b2ae27..ba36628f 100644 --- a/gemma/activations.h +++ b/gemma/activations.h @@ -529,6 +529,12 @@ struct Activations { s_ffw_in(num_layers, max_workers), s_ffw_hidden(num_layers, max_workers), s_ffw_out(num_layers, max_workers), + pooled(MatFactory("pooled", config.IsEmbedding() ? 1 : 0, + config.model_dim, ctx.allocator, + MatPadding::kPacked)), + h1(MatFactory("h1", config.IsEmbedding() ? 1 : 0, + config.EmbeddingProjDim(), ctx.allocator, + MatPadding::kPacked)), router_in(MatFactory("router_in", MoEBatchSize(moe_layer_config, batch_size), config.model_dim, ctx.allocator)), @@ -632,6 +638,10 @@ struct Activations { if (config.hc_mult > 1) { hc_mixes.AllocateAndAttachRowPtrs(row_ptrs); } + if (config.IsEmbedding()) { + pooled.AllocateAndAttachRowPtrs(row_ptrs); + h1.AllocateAndAttachRowPtrs(row_ptrs); + } if (config.is_encoder_decoder) { t5_encoder_q.AllocateAndAttachRowPtrs(row_ptrs); @@ -798,6 +808,10 @@ struct Activations { TensorStats s_ffw_hidden; // after Activation+gating TensorStats s_ffw_out; + // Embedding pooling and projection activations. + MatStorageT pooled; + MatStorageT h1; + // For MoE layers. These are used outside the expert-parallel loop: MatStorageT router_in; MatStorageT router_logits; // batch_size x num_experts diff --git a/gemma/configs.cc b/gemma/configs.cc index 9110eb58..e43df883 100644 --- a/gemma/configs.cc +++ b/gemma/configs.cc @@ -670,6 +670,48 @@ static ModelConfig ConfigGemma4_E4B() { return config; } +static LayerConfig LayerConfigEmbeddingGemma2_LM(size_t model_dim) { + LayerConfig config; + config.model_dim = model_dim; + config.ff_hidden_dim = 2048; + config.heads = 4; + config.kv_heads = 2; + config.qkv_dim = 256; + config.post_norm = PostNormType::Scale; + config.use_qk_norm = true; + config.norm_v = true; + config.ple_dim = 512; + return config; +} + +static ModelConfig ConfigEmbeddingGemma2_LM() { + ModelConfig config = ConfigBaseGemmaV4(); + config.display_name = "EmbeddingGemma2_LM"; + config.model = Model::EMBEDDING_GEMMA2_LM; + config.wrapping = PromptWrapping::GEMMA_IT; + config.model_dim = 512; + config.vocab_size = kGemmaV3VocabSize; + // MatMul row limit (model supports 8192). + config.max_seq_len = kMaxBatchSize; + config.ple_dim = 512; + config.num_layers = 24; + config.use_global_timescale = true; + config.query_scale = QueryScaleType::One; + config.secondary_eos_id = 1; + LayerConfig layer_config = LayerConfigEmbeddingGemma2_LM(config.model_dim); + config.layer_configs = {config.num_layers, layer_config}; + for (size_t i = 0; i < config.num_layers; ++i) { + if (i % 6 == 5) { + config.layer_configs[i].kv_heads = 1; + config.layer_configs[i].qkv_dim = 512; + } + } + // W = 513 gives [q - 512, q + 512] (|q - k| <= 512). + config.attention_window_sizes = RepeatedAttentionWindowSizes<24, 6>( + {513, 513, 513, 513, 513, config.max_seq_len}); + return config; +} + static LayerConfig LayerConfigDeepSeek4_Flash(size_t model_dim) { LayerConfig config; config.model_dim = model_dim; @@ -975,6 +1017,8 @@ static ModelConfig ConfigFromModel(Model model) { return ConfigGemma4_E4B(); case Model::GEMMA4_E4B_LM: return ConfigGemma4_E4B_LM(); + case Model::EMBEDDING_GEMMA2_LM: + return ConfigEmbeddingGemma2_LM(); default: HWY_ABORT("Model type %d unknown.", static_cast(model)); } @@ -1034,6 +1078,8 @@ const char* ModelPrefix(Model model) { return "gemma4-e4b"; case Model::GEMMA4_E4B_LM: return "gemma4-e4b-lm"; + case Model::EMBEDDING_GEMMA2_LM: + return "embeddinggemma2-lm"; default: HWY_ABORT("Model type %d unknown.", static_cast(model)); } @@ -1233,6 +1279,9 @@ Model DeduceModel(const Path& blob_path, size_t layers, int layer_types) { case 18: return Model::GEMMA3_270M; + case 24: + return Model::EMBEDDING_GEMMA2_LM; + case 26: if (layer_types & (kDeducedViT | kDeducedKqNorm)) { return Model::GEMMA3_1B; diff --git a/gemma/configs.h b/gemma/configs.h index f1f63510..0ae76a82 100644 --- a/gemma/configs.h +++ b/gemma/configs.h @@ -39,6 +39,8 @@ HWY_INLINE_VAR constexpr int kSkipKV = 1; HWY_INLINE_VAR constexpr size_t kMaxQKVDim = 1024; +constexpr size_t kEmbeddingGemma2Dim = 768; + #ifndef GEMMA_FUSED_FFN #define GEMMA_FUSED_FFN 1 #endif // !GEMMA_FUSED_FFN @@ -328,6 +330,7 @@ enum class Model { GEMMA4_E2B_LM = GEMMA4_2B_LM, GEMMA4_E4B, GEMMA4_E4B_LM, + EMBEDDING_GEMMA2_LM, kSentinel, }; @@ -349,6 +352,10 @@ static inline bool IsQwen3(Model model) { model == Model::QWEN3_4B; } +static inline bool IsEmbeddingGemma2(Model model) { + return model == Model::EMBEDDING_GEMMA2_LM; +} + static inline bool IsObsolete(Model model) { const size_t i = static_cast(model); if (i == 5 || i == 6 || i == 8 || i == 9) return true; @@ -849,7 +856,18 @@ struct ModelConfig : public IFields { } bool IsEOS(int id) const { return (id == eos_id || id == secondary_eos_id); } - bool IsEmbedding() const { return false; } + + bool IsEmbedding() const { + if (IsEmbeddingGemma2(model)) return true; + return false; + } + + size_t EmbeddingProjDim() const { + if (IsEmbeddingGemma2(model)) return kEmbeddingGemma2Dim; + return 0; + } + + bool HasPLETable() const { return ple_dim > 0 && !IsEmbeddingGemma2(model); } // Major version of the model family, reflecting architecture changes. This is // more convenient to compare than `Model` because that also includes the diff --git a/gemma/configs_test.cc b/gemma/configs_test.cc index c554a46d..a3cbd49e 100644 --- a/gemma/configs_test.cc +++ b/gemma/configs_test.cc @@ -239,4 +239,53 @@ TEST(ConfigsTest, Gemma4E4BQuantizerSpecifiers) { EXPECT_EQ(e4b_lm_pt.wrapping, PromptWrapping::GEMMA_PT); } +TEST(ConfigsTest, EmbeddingGemma2) { + const ModelConfig config(Model::EMBEDDING_GEMMA2_LM, Type::kBF16, + PromptWrapping::GEMMA_IT); + EXPECT_EQ(config.display_name, "EmbeddingGemma2_LM"); + EXPECT_FALSE(config.HasGemma4Vit()); + EXPECT_TRUE(IsEmbeddingGemma2(config.model)); + EXPECT_TRUE(config.IsEmbedding()); + EXPECT_EQ(config.EmbeddingProjDim(), kEmbeddingGemma2Dim); + EXPECT_EQ(config.num_layers, 24u); + EXPECT_EQ(config.model_dim, 512u); + EXPECT_EQ(config.ple_dim, 512u); + EXPECT_FALSE(config.HasPLETable()); + EXPECT_EQ(config.vocab_size, 262144u); + EXPECT_EQ(config.max_seq_len, kMaxBatchSize); + EXPECT_EQ(config.query_scale, QueryScaleType::One); + EXPECT_TRUE(config.use_global_timescale); + EXPECT_FLOAT_EQ(config.partial_rotary_factor, 1.0f); + EXPECT_EQ(config.NumPrefillLayers(), 24u); + + for (size_t i = 0; i < 24; ++i) { + const LayerConfig& lc = config.layer_configs[i]; + EXPECT_EQ(lc.ple_dim, 512u) << i; + EXPECT_EQ(lc.ff_hidden_dim, 2048u) << i; + EXPECT_EQ(lc.heads, 4u) << i; + EXPECT_EQ(lc.post_qk, PostQKType::Rope) << i; + EXPECT_TRUE(lc.use_qk_norm) << i; + EXPECT_TRUE(lc.norm_v) << i; + const bool is_global = (i % 6 == 5); + EXPECT_EQ(lc.kv_heads, is_global ? 1u : 2u) << i; + EXPECT_EQ(lc.qkv_dim, is_global ? 512u : 256u) << i; + EXPECT_EQ(config.attention_window_sizes[i], + is_global ? config.max_seq_len : 513u) + << i; + } + + const ModelConfig bf16_it("embeddinggemma2-lm-bf16-it"); + EXPECT_EQ(bf16_it.model, Model::EMBEDDING_GEMMA2_LM); + EXPECT_EQ(bf16_it.weight, Type::kBF16); + EXPECT_EQ(bf16_it.wrapping, PromptWrapping::GEMMA_IT); + + const ModelConfig sfp_pt("embeddinggemma2-lm-sfp-pt"); + EXPECT_EQ(sfp_pt.model, Model::EMBEDDING_GEMMA2_LM); + EXPECT_EQ(sfp_pt.weight, Type::kSFP); + EXPECT_EQ(sfp_pt.wrapping, PromptWrapping::GEMMA_PT); + + EXPECT_EQ(DeduceModel(Path("embeddinggemma2-lm-bf16-it.sbs"), 24, 0), + Model::EMBEDDING_GEMMA2_LM); +} + } // namespace gcpp diff --git a/gemma/embeddings_test.cc b/gemma/embeddings_test.cc new file mode 100644 index 00000000..3f9a19ab --- /dev/null +++ b/gemma/embeddings_test.cc @@ -0,0 +1,274 @@ +// Tests that batched `ComputeEmbeddingsT` matches `ComputeEmbeddingT` of each +// prompt on a small synthetic EmbeddingGemma 2 model. + +#include +#include +#include + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "compression/types.h" +#include "gemma/configs.h" +#include "gemma/gemma.h" +#include "gemma/gemma_args.h" +#include "gemma/weights.h" +#include "ops/matmul.h" +#include "util/mat.h" +#include "util/threading_context.h" +#include "hwy/aligned_allocator.h" +#include "hwy/base.h" + +#ifndef HWY_DISABLED_TARGETS +#define HWY_DISABLED_TARGETS GEMMA_DISABLED_TARGETS +#endif // HWY_DISABLED_TARGETS + +// clang-format off +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "gemma/embeddings_test.cc" // NOLINT +// clang-format on +#include "hwy/foreach_target.h" // IWYU pragma: keep +#include "hwy/highway.h" + +HWY_BEFORE_NAMESPACE(); +namespace gcpp { +namespace HWY_NAMESPACE { + +// Defined in gemma.cc. +void ComputeEmbeddingT(const RuntimeConfig& runtime_config_in, + const PromptTokens& prompt, const ModelConfig& config, + const InferenceArgs& inference, + const WeightsPtrs& weights, + hwy::AlignedVector& embedding, MatMulEnv& env); +void ComputeEmbeddingsT(const RuntimeConfig& runtime_config_in, + hwy::Span prompts, + const ModelConfig& config, + const InferenceArgs& inference, + const WeightsPtrs& weights, + std::vector>& embeddings, + MatMulEnv& env, EmbeddingsWorkspace* workspace); + +} // namespace HWY_NAMESPACE +} // namespace gcpp +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE +namespace gcpp { +namespace { + +HWY_EXPORT(ComputeEmbeddingT); +HWY_EXPORT(ComputeEmbeddingsT); + +bool IsUnused(const char* name) { + for (const char* prefix : {"att_ein", "qkv_ein", "gating_ein"}) { + if (strncmp(name, prefix, strlen(prefix)) == 0) return true; + } + return false; +} + +void AllocateRandom(MatPtr& mat, uint64_t seed, ThreadingContext& ctx, + std::vector& owners) { + if (mat.IsEmpty()) return; + mat.SetType(Type::kBF16); + mat.SetScale(1.0f); + owners.emplace_back(); + owners.back().AllocateFor(mat, ctx.allocator, MatPadding::kPacked); + BF16* base = reinterpret_cast(mat.RowBytes(0)); + float mean = 0.0f; + if (strncmp(mat.Name(), "skip_scale", 10) == 0) { + mean = 0.95f; + } else if (strncmp(mat.Name(), "query_norm", 10) == 0 || + strncmp(mat.Name(), "key_norm", 8) == 0) { + mean = -0.75f; // RMSNorm adds 1.0. + } + uint64_t s = seed * 0x9E3779B97F4A7C15ULL + 1; + for (size_t r = 0; r < mat.Rows(); ++r) { + BF16* row = base + r * mat.Stride(); + for (size_t c = 0; c < mat.Stride(); ++c) { + s ^= s >> 12; + s ^= s << 25; + s ^= s >> 27; + const int32_t v = static_cast((s * 0x2545F4914F6CDD1DULL) >> 48); + row[c] = hwy::BF16FromF32(mean + static_cast(v - 32768) * + (0.02f / 32768.0f)); + } + } +} + +struct TestModel { + explicit TestModel(ThreadingContext& ctx) + : config(Model::EMBEDDING_GEMMA2_LM, Type::kBF16, + PromptWrapping::GEMMA_IT) { + config.num_layers = 6; // 5 sliding-window + 1 global attention layer. + config.model_dim = 256; + config.ple_dim = 256; + config.vocab_size = 512; + config.layer_configs.resize(config.num_layers); + config.attention_window_sizes.resize(config.num_layers); + for (size_t i = 0; i < config.num_layers; ++i) { + LayerConfig& lc = config.layer_configs[i]; + lc.model_dim = config.model_dim; + lc.ple_dim = config.ple_dim; + lc.ff_hidden_dim = 512; + if (i + 1 < config.num_layers) config.attention_window_sizes[i] = 16; + } + weights = std::make_unique(config); + uint64_t seed = 1; + AllocateRandom(weights->embedder_input_embedding, ++seed, ctx, owners); + AllocateRandom(weights->final_norm_scale, ++seed, ctx, owners); + AllocateRandom(weights->ple_model_proj, ++seed, ctx, owners); + AllocateRandom(weights->ple_proj_norm, ++seed, ctx, owners); + AllocateRandom(weights->default_proj, ++seed, ctx, owners); + for (size_t i = 0; i < config.num_layers; ++i) { + LayerWeightsPtrs* layer = weights->GetLayer(i); + layer->ForEachTensor(nullptr, nullptr, [&](const TensorArgs& t) { + if (IsUnused(t.mat.Name())) return; + AllocateRandom(t.mat, ++seed, ctx, owners); + }); + } + } + + ModelConfig config; + std::unique_ptr weights; + std::vector owners; +}; + +double NormSquared(hwy::Span v) { + double sum = 0.0; + for (float x : v) sum += static_cast(x) * x; + return sum; +} + +double Cosine(hwy::Span a, hwy::Span b) { + HWY_ASSERT(a.size() == b.size()); + double ab = 0.0; + for (size_t i = 0; i < a.size(); ++i) { + ab += static_cast(a[i]) * b[i]; + } + const double aa = NormSquared(a); + const double bb = NormSquared(b); + HWY_ASSERT(aa > 0.0 && bb > 0.0); + return ab / std::sqrt(aa * bb); +} + +std::vector> MakePrompts(const std::vector& lengths, + size_t vocab_size) { + std::vector> prompts; + for (size_t p = 0; p < lengths.size(); ++p) { + std::vector prompt(lengths[p]); + for (size_t i = 0; i < lengths[p]; ++i) { + prompt[i] = + static_cast((i * 131 + p * 7919 + 7) % (vocab_size - 4)) + 2; + } + prompts.push_back(std::move(prompt)); + } + return prompts; +} + +void CheckBatchedMatchesSingle(const std::vector& lengths) { + ThreadingContext ctx{ThreadingArgs()}; + MatMulEnv env(ctx); + TestModel model(ctx); + InferenceArgs inference; + inference.seq_len = 2048; + RuntimeConfig runtime_config; + + const std::vector> storage = + MakePrompts(lengths, model.config.vocab_size); + std::vector prompts; + for (const std::vector& p : storage) prompts.emplace_back(p); + + // Reuses one workspace: grows from 1 prompt to all, then reuses for 2. + EmbeddingsWorkspace workspace; + std::vector> batched; + HWY_DYNAMIC_DISPATCH(ComputeEmbeddingsT)( + runtime_config, hwy::Span(prompts.data(), 1), + model.config, inference, *model.weights, batched, env, &workspace); + HWY_DYNAMIC_DISPATCH(ComputeEmbeddingsT)( + runtime_config, hwy::Span(prompts), model.config, + inference, *model.weights, batched, env, &workspace); + ASSERT_EQ(batched.size(), prompts.size()); + const Activations* acts_ptr = workspace.activations.get(); + const KVCache* kv_ptr = workspace.kv_caches.data(); + + std::vector> sub; + HWY_DYNAMIC_DISPATCH(ComputeEmbeddingsT)( + runtime_config, hwy::Span(prompts.data(), 2), + model.config, inference, *model.weights, sub, env, &workspace); + EXPECT_EQ(workspace.activations.get(), acts_ptr); + EXPECT_EQ(workspace.kv_caches.data(), kv_ptr); + EXPECT_GT(Cosine(batched[0], sub[0]), 0.9999); + EXPECT_GT(Cosine(batched[1], sub[1]), 0.9999); + + for (size_t p = 0; p < prompts.size(); ++p) { + hwy::AlignedVector single; + HWY_DYNAMIC_DISPATCH(ComputeEmbeddingT)(runtime_config, prompts[p], + model.config, inference, + *model.weights, single, env); + EXPECT_EQ(single.size(), kEmbeddingGemma2Dim); + EXPECT_EQ(batched[p].size(), kEmbeddingGemma2Dim); + EXPECT_NEAR(NormSquared(single), 1.0, 1e-5); + EXPECT_NEAR(NormSquared(batched[p]), 1.0, 1e-5); + EXPECT_GT(Cosine(single, batched[p]), 0.9999) + << "prompt " << p << " of length " << lengths[p]; + } + EXPECT_LT(Cosine(batched[0], batched[1]), 0.999); +} + +TEST(EmbeddingGemma2Test, BatchedMatchesSingleVariableLength) { + CheckBatchedMatchesSingle({5, 40, 17, 64, 1, 29}); +} + +// More rows than `kMaxBatchSize`, hence several groups. +TEST(EmbeddingGemma2Test, BatchedSeveralGroups) { + CheckBatchedMatchesSingle({1500, 1400, 1300, 7}); +} + +TEST(EmbeddingGemma2Test, MatryoshkaTruncation) { + ThreadingContext ctx{ThreadingArgs()}; + MatMulEnv env(ctx); + TestModel model(ctx); + InferenceArgs inference; + inference.seq_len = 256; + + const std::vector> storage = + MakePrompts({19, 37}, model.config.vocab_size); + std::vector prompts; + for (const std::vector& p : storage) prompts.emplace_back(p); + + hwy::AlignedVector full; + HWY_DYNAMIC_DISPATCH(ComputeEmbeddingT)(RuntimeConfig(), prompts[0], + model.config, inference, + *model.weights, full, env); + ASSERT_EQ(full.size(), kEmbeddingGemma2Dim); + + for (size_t dim : {size_t{128}, size_t{256}, size_t{512}, size_t{1024}}) { + const size_t expected_dim = HWY_MIN(dim, kEmbeddingGemma2Dim); + RuntimeConfig cfg; + cfg.embed_dim = dim; + + hwy::AlignedVector single; + HWY_DYNAMIC_DISPATCH(ComputeEmbeddingT)( + cfg, prompts[0], model.config, inference, *model.weights, single, env); + ASSERT_EQ(single.size(), expected_dim); + EXPECT_NEAR(NormSquared(single), 1.0, 1e-5); + EXPECT_GT(Cosine(single, hwy::Span(full.data(), expected_dim)), + 0.9999) + << "dim=" << dim; + + std::vector> batched; + HWY_DYNAMIC_DISPATCH(ComputeEmbeddingsT)( + cfg, hwy::Span(prompts), model.config, inference, + *model.weights, batched, env, /*workspace=*/nullptr); + ASSERT_EQ(batched[0].size(), expected_dim); + EXPECT_NEAR(NormSquared(batched[0]), 1.0, 1e-5); + EXPECT_GT(Cosine(single, batched[0]), 0.9999) << "dim=" << dim; + } +} + +} // namespace +} // namespace gcpp +#endif // HWY_ONCE diff --git a/gemma/gemma.cc b/gemma/gemma.cc index 1b419d82..02c3356d 100644 --- a/gemma/gemma.cc +++ b/gemma/gemma.cc @@ -150,10 +150,13 @@ HWY_NOINLINE void TransformerLayer(const size_t num_tokens, ResidualConnection(activations.ffw_out, activations.x, layer, /*is_attention=*/false, env.ctx); + const Model m = activations.attention.config.model; if (layer_config.ple_dim > 0) { + const size_t ple_rows = + IsEmbeddingGemma2(m) ? activations.x.Rows() : num_tokens; // 1. Gate: [batch, model_dim] @ [model_dim, ple_dim] -> [batch, ple_dim] // Use activations.x_bf to convert activations.x - for (size_t r = 0; r < num_tokens; ++r) { + for (size_t r = 0; r < ple_rows; ++r) { DemoteRowToBF16(activations.x.Row(r), activations.x_bf.Row(r), layer_config.model_dim); } @@ -166,8 +169,8 @@ HWY_NOINLINE void TransformerLayer(const size_t num_tokens, const size_t ple_dim = layer_config.ple_dim; const size_t layer_offset = layer_idx * ple_dim; - ParallelFor(Parallelism::kFlat, num_tokens, env.ctx, /*cluster_idx=*/0, - Callers::kActivationBatched, + ParallelFor(Parallelism::kFlat, ple_rows, env.ctx, + /*cluster_idx=*/0, Callers::kActivationBatched, [&](uint64_t token_idx, size_t worker) HWY_ATTR { BF16* g_row = activations.gate_out.Row(token_idx); const float* p_row = @@ -190,8 +193,8 @@ HWY_NOINLINE void TransformerLayer(const size_t num_tokens, // 4. Norm and Residual Add RMSNormInplaceBatched(layer.post_ple_ns, activations.ffw_out, env.ctx); - ParallelFor(Parallelism::kFlat, num_tokens, env.ctx, /*cluster_idx=*/0, - Callers::kOpsAddFromBatched, + ParallelFor(Parallelism::kFlat, ple_rows, env.ctx, + /*cluster_idx=*/0, Callers::kOpsAddFromBatched, [&](uint64_t token_idx, size_t worker) { AddFrom(activations.ffw_out.Row(token_idx), activations.x.Row(token_idx), layer_config.model_dim, @@ -199,7 +202,6 @@ HWY_NOINLINE void TransformerLayer(const size_t num_tokens, }); } - const Model m = activations.attention.config.model; if (layer.skip_scale.HasPtr() && m != Model::GEMMA4_26B_MOE) { float skip_scale_val = 1.0f; if (layer.skip_scale.GetType() == Type::kF32) { @@ -1092,6 +1094,8 @@ static HWY_NOINLINE void ComputePLEEmbeddings(size_t tbatch_size, }); }); + if (!weights.ple_embeddings.HasPtr()) return; + // 5. Add token embedding and apply input scale const float scale_input = 1.0f / sqrtf(2.0f); // Use pre-allocated activations.ple_token_emb @@ -1866,6 +1870,296 @@ void GenerateImageTokensT(const ModelConfig& config, timing_info.NotifyImageTokenDone(num_tokens); } +// Given the sum over `num_tokens` final-normed rows in `activations.pooled` +// row 0, computes their mean, applies the projection head, and writes the +// L2-normalized (Matryoshka-truncated) result to `embedding`. +static void PoolAndProjectEmbedding(const RuntimeConfig& runtime_config, + const ModelConfig& config, + const WeightsPtrs& weights, + size_t num_tokens, Activations& activations, + hwy::AlignedVector& embedding, + MatMulEnv& env) { + namespace hn = hwy::HWY_NAMESPACE; + using DF = hn::ScalableTag; + using VF = hn::Vec; + const DF df; + const size_t N = hn::Lanes(df); + const size_t model_dim = config.model_dim; + + float* HWY_RESTRICT pooled_ptr = activations.pooled.Row(0); + MulByConst(1.0f / static_cast(num_tokens), pooled_ptr, model_dim); + + // L2-normalizes the first `embed_dim` entries of `src` into `embedding`. + // Truncating before normalizing is what makes each shorter Matryoshka + // prefix a unit vector in its own right. + const auto normalize_prefix = [&](const float* HWY_RESTRICT src, + size_t full_dim) HWY_ATTR { + const size_t out_dim = runtime_config.embed_dim > 0 + ? HWY_MIN(runtime_config.embed_dim, full_dim) + : full_dim; + VF v_sum = hn::Zero(df); + size_t i = 0; + for (; i + N <= out_dim; i += N) { + const VF v = hn::Load(df, src + i); + v_sum = hn::MulAdd(v, v, v_sum); + } + if (i < out_dim) { + const VF v = hn::LoadN(df, src + i, out_dim - i); + v_sum = hn::MulAdd(v, v, v_sum); + } + const float l2_sum = hn::ReduceSum(df, v_sum); + // Otherwise the reciprocal below is inf/NaN and the failure is silent. + HWY_ASSERT(l2_sum > 0.0f); + embedding.resize(out_dim); + MulByConstTo(1.0f / sqrtf(l2_sum), src, embedding.data(), out_dim, env.ctx, + /*worker=*/0); + }; + + HWY_DASSERT(weights.default_proj.HasPtr()); + PROFILER_ZONE("Gen.Embedding.Proj"); + CallMatMul(activations.pooled, weights.default_proj, /*add=*/nullptr, env, + activations.h1); + normalize_prefix(activations.h1.Row(0), weights.default_proj.Rows()); +} + +// Returns `runtime_config_in` adapted for embedding prompts of at most +// `max_tokens` tokens, each in a single pass through the layers. +static RuntimeConfig EmbeddingRuntimeConfig( + const RuntimeConfig& runtime_config_in, size_t max_tokens) { + RuntimeConfig runtime_config = runtime_config_in; + runtime_config.stream_token = [](int, float) { return true; }; + runtime_config.prefill_tbatch_size = max_tokens; + // TiledAttention is the validated path for embedding prefill. + if (runtime_config.attention_impl == AttentionImpl::kFlash) { + runtime_config.attention_impl = AttentionImpl::kFlashTransposedQs; + } + return runtime_config; +} + +// Writes the sum of `num_rows` rows of `x`, starting at `row` and `stride` +// rows apart, to row 0 of `pooled`. +static void SumRows(const MatStorageT& x, size_t row, size_t stride, + size_t num_rows, MatStorageT& pooled, + ThreadingContext& ctx) { + PROFILER_ZONE("Gen.Embedding.Pool"); + float* HWY_RESTRICT sum = pooled.Row(0); + ZeroInit(pooled); + for (size_t i = 0; i < num_rows; ++i) { + AddFrom(x.Row(row + i * stride), sum, x.Cols(), ctx, /*worker=*/0); + } +} + +void ComputeEmbeddingT(const RuntimeConfig& runtime_config_in, + const PromptTokens& prompt, const ModelConfig& config, + const InferenceArgs& inference, + const WeightsPtrs& weights, + hwy::AlignedVector& embedding, MatMulEnv& env) { + HWY_ASSERT(config.IsEmbedding()); + const size_t num_tokens = prompt.size(); + HWY_ASSERT(num_tokens > 0); + + const RuntimeConfig runtime_config = + EmbeddingRuntimeConfig(runtime_config_in, num_tokens); + + KVCache kv_cache(config, inference, runtime_config, env.ctx.allocator); + HWY_ASSERT(num_tokens <= kv_cache.SeqLen()); + + Activations activations(runtime_config, config, num_tokens, kv_cache.SeqLen(), + env.ctx, env.row_ptrs); + + // A non-zero `prefix_end` makes attention over the prompt bidirectional. + const size_t prefix_end = num_tokens; + + AllQueries all_queries(prompt, /*pos=*/0, prefix_end, + hwy::Span(&kv_cache, 1)); + QBatch qbatch(/*start=*/0, /*max_size=*/1, all_queries); + hwy::BitSet4096<> non_eos; + + { + GCPP_ZONE(env.ctx, hwy::Profiler::GlobalIdx(), Zones::kGenEmbedding); + + PrefillTBatch(config, runtime_config, weights, activations, qbatch, env, + non_eos); + + RMSNormInplaceBatched(weights.final_norm_scale, activations.x, env.ctx); + + SumRows(activations.x, /*row=*/0, /*stride=*/1, num_tokens, + activations.pooled, env.ctx); + PoolAndProjectEmbedding(runtime_config, config, weights, num_tokens, + activations, embedding, env); + } + + // Print results if profiler is enabled. + env.ctx.profiler.PrintResults(); +} + +// Prompts that pass through the layers together. +struct EmbeddingGroup { + size_t begin; // Index of its first prompt in `order`. + size_t size; // Number of prompts. + size_t num_tokens; // Of its longest (= first) prompt. +}; + +// Sorts the prompts by decreasing length into `order` and splits them into +// groups of at most `kMaxBatchSize` rows. Groups are balanced, e.g. 32 + 32 +// rather than 56 + 8 prompts of equal length. +static std::vector GroupByLength( + hwy::Span prompts, std::vector& order) { + order.resize(prompts.size()); + std::iota(order.begin(), order.end(), size_t{0}); + std::stable_sort(order.begin(), order.end(), [&](size_t a, size_t b) { + return prompts[a].size() > prompts[b].size(); + }); + std::vector groups; + for (size_t i = 0; i < prompts.size();) { + const size_t num_tokens = prompts[order[i]].size(); + const size_t max_size = kMaxBatchSize / num_tokens; + const size_t remaining = prompts.size() - i; + const size_t size = + remaining <= max_size + ? remaining + : hwy::DivCeil(remaining, hwy::DivCeil(remaining, max_size)); + groups.push_back(EmbeddingGroup{i, size, num_tokens}); + i += size; + } + return groups; +} + +// (Re)allocates the buffers of `ws` unless they suffice for `groups`. +static void ReserveEmbeddingsWorkspace( + const std::vector& groups, const ModelConfig& config, + const InferenceArgs& inference, const RuntimeConfig& runtime_config, + MatMulEnv& env, EmbeddingsWorkspace& ws) { + const size_t max_tokens = groups[0].num_tokens; + size_t max_size = 0; + size_t max_rows = 0; + for (const EmbeddingGroup& group : groups) { + max_size = HWY_MAX(max_size, group.size); + max_rows = HWY_MAX(max_rows, group.size * group.num_tokens); + } + if (ws.activations && ws.num_queries >= max_size && + ws.num_tokens >= max_tokens && ws.num_rows >= max_rows && + ws.attention_impl == runtime_config.attention_impl) { + return; + } + ws.activations.reset(); + ws.kv_caches.clear(); + // Embeddings do not generate, hence the KV caches only need to hold the + // prompt. Allocating and zeroing `inference.seq_len` would dominate. + InferenceArgs kv_inference = inference; + kv_inference.seq_len = max_tokens; + ws.kv_caches.reserve(max_size); + for (size_t qi = 0; qi < max_size; ++qi) { + ws.kv_caches.emplace_back(config, kv_inference, runtime_config, + env.ctx.allocator); + } + HWY_ASSERT(max_tokens <= ws.kv_caches[0].SeqLen()); + ws.activations = std::make_unique( + runtime_config, config, max_rows, ws.kv_caches[0].SeqLen(), env.ctx, + env.row_ptrs); + ws.num_queries = max_size; + ws.num_tokens = max_tokens; + ws.num_rows = max_rows; + ws.attention_impl = runtime_config.attention_impl; +} + +// Embeds `num_tokens` tokens of each of `prompts` into `activations.x`, with +// token `ti` of prompt `qi` in row `ti * prompts.size() + qi`, which is the +// `QBatch` layout of `TransformerLayer`. Shorter prompts are padded at the end. +static void EmbedGroup(hwy::Span prompts, size_t num_tokens, + const ModelConfig& config, const WeightsPtrs& weights, + Activations& activations, ThreadingContext& ctx) { + const size_t num_rows = num_tokens * prompts.size(); + activations.SetBatchSize(num_rows); + activations.token_ids.resize(num_rows); + activations.is_padding.assign(num_rows, 0); + for (size_t ti = 0; ti < num_tokens; ++ti) { + for (size_t qi = 0; qi < prompts.size(); ++qi) { + const size_t row = ti * prompts.size() + qi; + const bool is_padding = ti >= prompts[qi].size(); + activations.is_padding[row] = is_padding; + // Padding: any valid token works because its rows are not pooled. + const int token = prompts[qi][is_padding ? 0 : ti]; + activations.token_ids[row] = token; + EmbedMMToken(token, row, /*pos=*/ti, /*pos_in_prompt=*/ti, config, + weights, activations.x, ctx, /*image_tokens=*/nullptr, + /*image_token_position=*/0); + } + } +} + +// Batched `ComputeEmbeddingT`: prompts of similar length pass through the +// layers together, see `GroupByLength`. The padding of shorter prompts does +// not change their embeddings: `prefix_end` limits bidirectional attention to +// the real tokens, and only real tokens are pooled. +void ComputeEmbeddingsT(const RuntimeConfig& runtime_config_in, + hwy::Span prompts, + const ModelConfig& config, + const InferenceArgs& inference, + const WeightsPtrs& weights, + std::vector>& embeddings, + MatMulEnv& env, EmbeddingsWorkspace* workspace) { + HWY_ASSERT(config.IsEmbedding()); + HWY_ASSERT(prompts.size() != 0); + // As for `ComputeEmbeddingT`, each prompt passes through the layers at once. + for (const PromptTokens& prompt : prompts) { + HWY_ASSERT(prompt.size() != 0 && prompt.size() <= kMaxBatchSize); + } + + std::vector order; + const std::vector groups = GroupByLength(prompts, order); + const RuntimeConfig runtime_config = + EmbeddingRuntimeConfig(runtime_config_in, groups[0].num_tokens); + EmbeddingsWorkspace local_workspace; + EmbeddingsWorkspace& ws = workspace ? *workspace : local_workspace; + ReserveEmbeddingsWorkspace(groups, config, inference, runtime_config, env, + ws); + Activations& activations = *ws.activations; + + GCPP_ZONE(env.ctx, hwy::Profiler::GlobalIdx(), Zones::kGenEmbedding); + embeddings.resize(prompts.size()); + std::vector group_prompts; + std::vector prefix_end; + for (const EmbeddingGroup& group : groups) { + group_prompts.clear(); + prefix_end.clear(); + for (size_t qi = 0; qi < group.size; ++qi) { + group_prompts.push_back(prompts[order[group.begin + qi]]); + prefix_end.push_back(group_prompts.back().size()); + } + const hwy::Span span(group_prompts.data(), group.size); + AllQueries all_queries( + span, hwy::Span(ws.kv_caches.data(), group.size), + hwy::Span(prefix_end.data(), group.size)); + QBatch qbatch(/*start=*/0, /*max_size=*/group.size, all_queries); + + EmbedGroup(span, group.num_tokens, config, weights, activations, env.ctx); + if (config.ple_dim > 0) { + ComputePLEEmbeddings(activations.x.Rows(), activations.token_ids, config, + weights, activations, env); + } + for (size_t layer_idx = 0; layer_idx < config.layer_configs.size(); + ++layer_idx) { + TransformerLayer(group.num_tokens, layer_idx, + *weights.GetLayer(layer_idx), activations, qbatch, env); + } + RMSNormInplaceBatched(weights.final_norm_scale, activations.x, env.ctx); + + for (size_t qi = 0; qi < group.size; ++qi) { + const size_t num_tokens = group_prompts[qi].size(); + SumRows(activations.x, /*row=*/qi, /*stride=*/group.size, num_tokens, + activations.pooled, env.ctx); + PoolAndProjectEmbedding(runtime_config, config, weights, num_tokens, + activations, embeddings[order[group.begin + qi]], + env); + } + } + activations.is_padding.clear(); + + // Print results if profiler is enabled. + env.ctx.profiler.PrintResults(); +} + // NOLINTNEXTLINE(google-readability-namespace-comments) } // namespace HWY_NAMESPACE } // namespace gcpp @@ -1876,6 +2170,8 @@ namespace gcpp { HWY_EXPORT(GenerateSingleT); HWY_EXPORT(GenerateBatchT); HWY_EXPORT(GenerateImageTokensT); +HWY_EXPORT(ComputeEmbeddingT); +HWY_EXPORT(ComputeEmbeddingsT); Gemma::Gemma(const GemmaArgs& args, ThreadingContext& ctx) : reader_(args.loader.weights), @@ -1947,6 +2243,40 @@ void Gemma::GenerateImageTokens(const RuntimeConfig& runtime_config, env.ctx.pools.MaybeStopSpinning(runtime_config.use_spinning); } +void Gemma::ComputeEmbedding(const PromptTokens& prompt, + hwy::AlignedVector& embedding, + MatMulEnv& env) const { + RuntimeConfig runtime_config; + inference_.CopyTo(runtime_config); + ComputeEmbedding(runtime_config, prompt, embedding, env); +} + +void Gemma::ComputeEmbedding(const RuntimeConfig& runtime_config, + const PromptTokens& prompt, + hwy::AlignedVector& embedding, + MatMulEnv& env) const { + env.ctx.pools.MaybeStartSpinning(runtime_config.use_spinning); + + HWY_DYNAMIC_DISPATCH(ComputeEmbeddingT)(runtime_config, prompt, + model_.Config(), inference_, + weights_, embedding, env); + + env.ctx.pools.MaybeStopSpinning(runtime_config.use_spinning); +} + +void Gemma::ComputeEmbeddings( + const RuntimeConfig& runtime_config, hwy::Span prompts, + std::vector>& embeddings, MatMulEnv& env, + EmbeddingsWorkspace* workspace) const { + env.ctx.pools.MaybeStartSpinning(runtime_config.use_spinning); + + HWY_DYNAMIC_DISPATCH(ComputeEmbeddingsT)( + runtime_config, prompts, model_.Config(), inference_, weights_, + embeddings, env, workspace); + + env.ctx.pools.MaybeStopSpinning(runtime_config.use_spinning); +} + ContinuousQBatch::ContinuousQBatch(size_t max_size, AllQueries& queries) : QBatch(0, max_size, queries) { for (size_t i = start_; i < queries_.NumQueries(); ++i) { diff --git a/gemma/gemma.h b/gemma/gemma.h index 1338a2a4..8223f779 100644 --- a/gemma/gemma.h +++ b/gemma/gemma.h @@ -150,6 +150,18 @@ struct TimingInfo { int verbosity = 0; }; +// Buffers for `Gemma::ComputeEmbeddings`, reused by subsequent calls that fit +// within their capacity. Use one per concurrent caller. +struct EmbeddingsWorkspace { + // Capacity: prompts per group, tokens per prompt, and rows per layer. + size_t num_queries = 0; + size_t num_tokens = 0; + size_t num_rows = 0; + AttentionImpl attention_impl = AttentionImpl::kSentinel; + std::vector kv_caches; + std::unique_ptr activations; +}; + // After construction, all methods are const and thread-compatible if using // separate `ThreadingContext` and `MatMulEnv` for each concurrent `Generate`. class Gemma { @@ -199,6 +211,30 @@ class Gemma { const Image& image, ImageTokens& image_tokens, MatMulEnv& env, TimingInfo& timing_info) const; + // Computes normalized text embedding for embedding models (e.g. + // EMBEDDING_GEMMA2_LM). + void ComputeEmbedding(const PromptTokens& prompt, + hwy::AlignedVector& embedding, + MatMulEnv& env) const; + void ComputeEmbedding(const RuntimeConfig& runtime_config, + const PromptTokens& prompt, + hwy::AlignedVector& embedding, + MatMulEnv& env) const; + // Computes one embedding per prompt. Prompts may differ in length. Their + // tokens pass through each layer together, which amortizes reading the + // weights over all prompts. + // Prompts are sorted by length and grouped such that each layer sees at + // most `kMaxBatchSize` rows; shorter prompts are padded to the longest in + // their group, and the padding does not affect the embeddings. As for + // `ComputeEmbedding`, each prompt must have at most `kMaxBatchSize` tokens. + // If `workspace` is non-null, its buffers are reused if large enough, + // otherwise (re)allocated. + void ComputeEmbeddings(const RuntimeConfig& runtime_config, + hwy::Span prompts, + std::vector>& embeddings, + MatMulEnv& env, + EmbeddingsWorkspace* workspace = nullptr) const; + private: BlobReader reader_; ModelStore model_; diff --git a/gemma/gemma_args.h b/gemma/gemma_args.h index d1934d2a..d119403a 100644 --- a/gemma/gemma_args.h +++ b/gemma/gemma_args.h @@ -209,6 +209,10 @@ struct RuntimeConfig { // `max_generated_tokens == 0`, `!use_continuous_batching`, and no MTP // decoding (whose prefill pairs each token with its successor). bool prefill_last_token = false; + + // Embedding output dimension truncation (0 = full dimension). Only consulted + // by `Gemma::ComputeEmbedding`. + size_t embed_dim = 0; }; struct InferenceArgs : public ArgsBase { @@ -244,6 +248,7 @@ struct InferenceArgs : public ArgsBase { bool use_mtp; size_t mtp_draft_horizon; float mtp_confidence_threshold; + size_t embed_dim; template void ForEach(const Visitor& visitor) { @@ -314,6 +319,8 @@ struct InferenceArgs : public ArgsBase { visitor(mtp_confidence_threshold, "mtp_confidence_threshold", 0.0f, "Minimum top-1 probability threshold to accept MTP drafts (default: 0.0)", 2); + visitor(embed_dim, "embed_dim", size_t{0}, + "Truncate embedding to this dimension (0 = full dimension).", 2); } void CopyTo(RuntimeConfig& runtime_config) const { @@ -350,6 +357,7 @@ struct InferenceArgs : public ArgsBase { runtime_config.use_mtp = use_mtp; runtime_config.mtp_draft_horizon = mtp_draft_horizon; runtime_config.mtp_confidence_threshold = mtp_confidence_threshold; + runtime_config.embed_dim = embed_dim; } }; diff --git a/gemma/run.cc b/gemma/run.cc index 65f714b5..34170dfa 100644 --- a/gemma/run.cc +++ b/gemma/run.cc @@ -224,6 +224,24 @@ void ReplGemma(const GemmaArgs& args, const Gemma& gemma, KVCache& kv_cache, } } + if (config.IsEmbedding()) { + prompt.clear(); + HWY_ASSERT(gemma.Tokenizer().Encode(prompt_string, &prompt)); + if (IsEmbeddingGemma2(config.model)) { + prompt.insert(prompt.begin(), BOS_ID); + prompt.push_back(config.eos_id); + } + prompt_size = prompt.size(); + hwy::AlignedVector embedding; + gemma.ComputeEmbedding(runtime_config, prompt, embedding, env); + std::cout << "[ Embedding: size " << embedding.size() << " ]\n"; + for (size_t i = 0; i < embedding.size(); ++i) { + std::cout << embedding[i] << (i + 1 < embedding.size() ? " " : "\n"); + } + if (!is_interactive) break; + continue; + } + // Generate until EOS or max_generated_tokens. if (inference.verbosity >= 1) { std::cerr << "\n[ Reading prompt ] " << std::flush; diff --git a/gemma/tensor_info.cc b/gemma/tensor_info.cc index 9841b5a3..6483e97e 100644 --- a/gemma/tensor_info.cc +++ b/gemma/tensor_info.cc @@ -84,6 +84,15 @@ void TensorInfoRegistry::AddModelTensors(const ModelConfig& config) { .min_size = Type::kBF16, }); } + if (IsEmbeddingGemma2(config.model)) { + Add(no_suffix, { + .base_name = "default_proj", + .source_names = {"embedding_projection.weight"}, + .axes = {0, 1}, + .shape = {config.EmbeddingProjDim(), config.model_dim}, + .min_size = Type::kBF16, + }); + } Add(no_suffix, { .base_name = "enc_norm_bias", .source_names = {"img/Transformer/encoder_norm/bias"}, @@ -150,16 +159,17 @@ void TensorInfoRegistry::AddModelTensors(const ModelConfig& config) { }); // Per-Layer Embedding (PLE) model-level tensors. if (config.ple_dim > 0) { - Add(no_suffix, - { - .base_name = "ple_embeddings", - .source_names = {"embedder/per_layer_embeddings"}, - .preshape = {config.vocab_size, - config.num_layers * config.ple_dim}, - .axes = {0, 1}, - .shape = {config.vocab_size, - config.num_layers * config.ple_dim}, - }); + if (config.HasPLETable()) { + Add(no_suffix, + { + .base_name = "ple_embeddings", + .source_names = {"embedder/per_layer_embeddings"}, + .preshape = {config.vocab_size, + config.num_layers * config.ple_dim}, + .axes = {0, 1}, + .shape = {config.vocab_size, config.num_layers * config.ple_dim}, + }); + } Add(no_suffix, { .base_name = "ple_model_proj", diff --git a/gemma/tensor_info_test.cc b/gemma/tensor_info_test.cc index b0c6d2c5..c3fae54c 100644 --- a/gemma/tensor_info_test.cc +++ b/gemma/tensor_info_test.cc @@ -173,5 +173,45 @@ TEST(TensorInfoRegistryTest, Gemma4E4BShapes) { EXPECT_EQ(linear_shared->shape[1], kFfHiddenDim); // 10240 } +TEST(TensorInfoRegistryTest, EmbeddingGemma2Shapes) { + const ModelConfig config(Model::EMBEDDING_GEMMA2_LM, Type::kBF16, + PromptWrapping::GEMMA_IT); + EXPECT_TRUE(config.vit_config.layer_configs.empty()); + const TensorInfoRegistry tensors(config); + + const TensorInfo* default_proj = tensors.Find("default_proj"); + ASSERT_NE(default_proj, nullptr); + EXPECT_EQ(default_proj->shape, (std::vector{768, 512})); + EXPECT_EQ(default_proj->axes, (std::vector{0, 1})); + EXPECT_EQ(default_proj->source_names, + (std::vector{"embedding_projection.weight"})); + EXPECT_EQ(default_proj->min_size, Type::kBF16); + + EXPECT_EQ(tensors.Find("ple_embeddings"), nullptr); + const TensorInfo* ple_model_proj = tensors.Find("ple_model_proj"); + ASSERT_NE(ple_model_proj, nullptr); + EXPECT_EQ(ple_model_proj->shape, (std::vector{24 * 512, 512})); + const TensorInfo* ple_proj_norm = tensors.Find("ple_proj_norm"); + ASSERT_NE(ple_proj_norm, nullptr); + EXPECT_EQ(ple_proj_norm->shape, (std::vector{512})); + + const TensorInfo* qkv1_local = tensors.Find("qkv1_w_0"); + ASSERT_NE(qkv1_local, nullptr); + EXPECT_EQ(qkv1_local->shape, (std::vector{4 * 256, 512})); + const TensorInfo* qkv2_local = tensors.Find("qkv2_w_0"); + ASSERT_NE(qkv2_local, nullptr); + EXPECT_EQ(qkv2_local->shape, (std::vector{2 * 2 * 256, 512})); + const TensorInfo* ple_gate = tensors.Find("ple_gate_0"); + ASSERT_NE(ple_gate, nullptr); + EXPECT_EQ(ple_gate->shape, (std::vector{512, 512})); + + const TensorInfo* qkv1_global = tensors.Find("qkv1_w_5"); + ASSERT_NE(qkv1_global, nullptr); + EXPECT_EQ(qkv1_global->shape, (std::vector{4 * 512, 512})); + const TensorInfo* qkv2_global = tensors.Find("qkv2_w_5"); + ASSERT_NE(qkv2_global, nullptr); + EXPECT_EQ(qkv2_global->shape, (std::vector{2 * 1 * 512, 512})); +} + } // namespace } // namespace gcpp diff --git a/gemma/tokenizer.cc b/gemma/tokenizer.cc index a4f60636..f06d8c86 100644 --- a/gemma/tokenizer.cc +++ b/gemma/tokenizer.cc @@ -73,7 +73,8 @@ GemmaChatTemplate::GemmaChatTemplate(const GemmaTokenizer& tokenizer, sot_model_ = {105, 4368, 107, 100, 45518, 107, 101}; eot_ = {106, 107}; } else if (model == Model::GEMMA4_2B || model == Model::GEMMA4_2B_LM || - model == Model::GEMMA4_E4B || model == Model::GEMMA4_E4B_LM) { + model == Model::GEMMA4_E4B || model == Model::GEMMA4_E4B_LM || + IsEmbeddingGemma2(model)) { sot_user_ = {105, 2364, 107}; sot_model_ = {105, 4368, 107}; eot_ = {106, 107}; diff --git a/gemma/weights.h b/gemma/weights.h index 2a644db1..bb9e006e 100644 --- a/gemma/weights.h +++ b/gemma/weights.h @@ -664,6 +664,7 @@ struct WeightsPtrs { finder_("", tensors_), // no suffix because these are per-model. embedder_input_embedding(finder_("c_embedding")), final_norm_scale(finder_("c_final_norm")), + default_proj(finder_("default_proj")), lm_head(finder_("lm_head")), hc_head_fn(finder_("hc_head_fn")), hc_head_base(finder_("hc_head_base")), @@ -740,6 +741,7 @@ struct WeightsPtrs { // TODO: switch to SFP? MatPtr embedder_input_embedding; MatPtr final_norm_scale; // at least BF16. + MatPtr default_proj; // embedding projection head MatPtr lm_head; // untied output head (DeepSeek); empty if tied. // mHC head collapse (DeepSeek V4; empty unless hc_mult > 1). MatPtr hc_head_fn; // [hc_mult, hc_mult * model_dim] f32 @@ -842,6 +844,9 @@ struct WeightsPtrs { func(TENSOR_ARGS(embedder_input_embedding, kMustRead)); func(TENSOR_ARGS(final_norm_scale, kMustRead)); + if (IsEmbeddingGemma2(config_.model)) { + func(TENSOR_ARGS(default_proj, kMustRead)); + } if (config_.HasLmHead()) { func(TENSOR_ARGS(lm_head, kMustRead)); } @@ -894,7 +899,9 @@ struct WeightsPtrs { } if (config_.ple_dim > 0) { - func(TENSOR_ARGS(ple_embeddings, kMustRead)); + if (config_.HasPLETable()) { + func(TENSOR_ARGS(ple_embeddings, kMustRead)); + } func(TENSOR_ARGS(ple_model_proj, kMustRead)); func(TENSOR_ARGS(ple_proj_norm, kMustRead)); } diff --git a/python/configs.cc b/python/configs.cc index e264f451..db373f55 100644 --- a/python/configs.cc +++ b/python/configs.cc @@ -116,6 +116,7 @@ PYBIND11_MODULE(configs, py_module) { .value("GEMMA4_2B_LM", gcpp::Model::GEMMA4_2B_LM) .value("GEMMA4_E4B", gcpp::Model::GEMMA4_E4B) .value("GEMMA4_E4B_LM", gcpp::Model::GEMMA4_E4B_LM) + .value("EMBEDDING_GEMMA2_LM", gcpp::Model::EMBEDDING_GEMMA2_LM) // Insert new models above this line. ; // NOLINT diff --git a/python/convert_from_safetensors.py b/python/convert_from_safetensors.py index 1e2dbe3c..ead5c25d 100644 --- a/python/convert_from_safetensors.py +++ b/python/convert_from_safetensors.py @@ -219,6 +219,7 @@ def _is_bf16_param(param_name: str) -> bool: "pre_", "post_", "c_", + "default_proj", "img_head_kernel", "query_norm", "key_norm", @@ -923,6 +924,8 @@ def export_gemma4_lm_sbs( llm_prefix = "model." elif "language_model.embed_tokens.weight" in params: llm_prefix = "language_model." + elif "embed_tokens.weight" in params: + llm_prefix = "" else: raise ValueError("Could not locate embed_tokens.weight in checkpoint.") @@ -931,7 +934,12 @@ def export_gemma4_lm_sbs( num_layers = len( set([k for k in params.keys() if k.endswith("input_layernorm.weight")]) ) - has_ple = f"{llm_prefix}embed_tokens_per_layer.weight" in params + ple_prefix = ( + f"{llm_prefix}ple." + if f"{llm_prefix}ple.per_layer_model_projection.weight" in params + else llm_prefix + ) + has_ple = f"{ple_prefix}per_layer_model_projection.weight" in params print( f"Gemma4 LM: vocab={vocab_size} dim={model_dim} layers={num_layers} " @@ -948,6 +956,7 @@ def export_gemma4_lm_sbs( else: target_weight_type = configs.Type.kSFP + is_eg2 = model_specifier.startswith("embeddinggemma2-") writer = compression.SbsWriter(sbs_file) metadata = [] scales = {} @@ -963,7 +972,8 @@ def add_data(param_name, data, expected_shape, sbs_name, layer_index=None): assert isinstance(data, torch.Tensor) data = data.to(torch.float32).numpy() data = np.array(data) - if "norm" in sbs_name or "_ns" in sbs_name: + is_norm = "norm" in sbs_name or "_ns" in sbs_name + if is_norm: data = data - 1.0 if layer_index is not None: @@ -975,14 +985,14 @@ def add_data(param_name, data, expected_shape, sbs_name, layer_index=None): both_names = param_name + "::" + sbs_name metadata.append((both_names, data.dtype, data.shape, scale)) - if _is_float_param(sbs_name): + if (is_eg2 and is_norm) or _is_float_param(sbs_name): packed = configs.Type.kF32 elif _is_bf16_param(sbs_name): packed = configs.Type.kBF16 else: packed = target_weight_type if packed == configs.Type.kSFP: - assert scale == 1.0, f"Scale for {both_names} is not 1.0" + assert is_eg2 or scale == 1.0, f"Scale for {both_names} is not 1.0" scales[sbs_name] = scale sys.stdout.flush() @@ -1062,25 +1072,35 @@ def add_gating_einsum(i): "c_final_norm", ) - if has_ple: - ple_weight = params[f"{llm_prefix}embed_tokens_per_layer.weight"] - ple_total_dim = ple_weight.shape[1] - ple_dim = ple_total_dim // num_layers + if f"{llm_prefix}embedding_projection.weight" in params: + proj = params.pop(f"{llm_prefix}embedding_projection.weight") add_data( - f"{llm_prefix}embed_tokens_per_layer.weight", - params.pop(f"{llm_prefix}embed_tokens_per_layer.weight"), - (vocab_size, ple_total_dim), - "ple_embeddings", + f"{llm_prefix}embedding_projection.weight", + proj, + (proj.shape[0], model_dim), + "default_proj", ) + + if has_ple: + ple_proj_weight = params[f"{ple_prefix}per_layer_model_projection.weight"] + ple_total_dim = ple_proj_weight.shape[0] + ple_dim = ple_total_dim // num_layers + if f"{llm_prefix}embed_tokens_per_layer.weight" in params: + add_data( + f"{llm_prefix}embed_tokens_per_layer.weight", + params.pop(f"{llm_prefix}embed_tokens_per_layer.weight"), + (vocab_size, ple_total_dim), + "ple_embeddings", + ) add_data( - f"{llm_prefix}per_layer_model_projection.weight", - params.pop(f"{llm_prefix}per_layer_model_projection.weight"), + f"{ple_prefix}per_layer_model_projection.weight", + params.pop(f"{ple_prefix}per_layer_model_projection.weight"), (ple_total_dim, model_dim), "ple_model_proj", ) add_data( - f"{llm_prefix}per_layer_projection_norm.weight", - params.pop(f"{llm_prefix}per_layer_projection_norm.weight"), + f"{ple_prefix}per_layer_projection_norm.weight", + params.pop(f"{ple_prefix}per_layer_projection_norm.weight"), (ple_dim,), "ple_proj_norm", ) @@ -1127,18 +1147,19 @@ def add_layer_tensor(layer_idx, suffix, shape, sbs_name): break if has_ple: - gate_dim = params[ - f"{llm_prefix}layers.{i}.per_layer_input_gate.weight" - ].shape[0] - add_layer_tensor( - i, "per_layer_input_gate.weight", (gate_dim, model_dim), "ple_gate" - ) - add_layer_tensor( - i, "per_layer_projection.weight", (model_dim, gate_dim), "ple_proj" - ) - add_layer_tensor( - i, "post_per_layer_input_norm.weight", (model_dim,), "post_ple_ns" + ple_gate = "per_layer_input_gate.weight" + ple_p = ( + "ple_block." + if f"{llm_prefix}layers.{i}.ple_block.{ple_gate}" in params + else "" ) + gate_dim = params[f"{llm_prefix}layers.{i}.{ple_p}{ple_gate}"].shape[0] + for name, shape, sbs_name in ( + (ple_gate, (gate_dim, model_dim), "ple_gate"), + ("per_layer_projection.weight", (model_dim, gate_dim), "ple_proj"), + ("post_per_layer_input_norm.weight", (model_dim,), "post_ple_ns"), + ): + add_layer_tensor(i, ple_p + name, shape, sbs_name) if params: print(f"WARNING: leftover params not consumed: {list(params.keys())[:10]}") @@ -1768,7 +1789,9 @@ def main(argv: Sequence[str]) -> None: export_gemma3_lm_sbs( model_specifier, load_path, tokenizer_file, metadata_file, sbs_file ) - elif model_specifier.startswith("gemma4-") and "-lm-" in model_specifier: + elif ( + model_specifier.startswith("gemma4-") and "-lm-" in model_specifier + ) or model_specifier.startswith("embeddinggemma2-lm-"): export_gemma4_lm_sbs( model_specifier, load_path, tokenizer_file, metadata_file, sbs_file ) @@ -1788,8 +1811,8 @@ def main(argv: Sequence[str]) -> None: else: raise app.UsageError( f"Unsupported model_specifier {model_specifier!r}. Expected a " - "'paligemma*', 'gemma3-*-lm-*', 'gemma4-*-lm-*', 'qwen3-*', or " - "'t5gemma*' specifier." + "'paligemma*', 'gemma3-*-lm-*', 'gemma4-*-lm-*', " + "'embeddinggemma2-lm-*', 'qwen3-*', or 't5gemma*' specifier." ) diff --git a/util/zones.cc b/util/zones.cc index 9cd3475f..ddff9678 100644 --- a/util/zones.cc +++ b/util/zones.cc @@ -45,6 +45,8 @@ const char* ZoneName(Zones zone) { return "Gen.Attention.SumHeads"; case Zones::kGenEmbed: return "Gen.Embed"; + case Zones::kGenEmbedding: + return "Gen.Embedding"; case Zones::kGenEmbeddingMatmul: return "Gen.EmbeddingMatmul"; case Zones::kGenFFW: @@ -134,6 +136,7 @@ hwy::ProfilerFlags ZoneFlags(Zones zone) { case Zones::kGenAttentionDotSoftmaxWeightedSumInclusive: case Zones::kGenAttentionSumHeads: case Zones::kGenEmbed: + case Zones::kGenEmbedding: case Zones::kGenEmbeddingMatmul: case Zones::kGenFFW: case Zones::kGenImageTokens: diff --git a/util/zones.h b/util/zones.h index 533182a9..656a9cf5 100644 --- a/util/zones.h +++ b/util/zones.h @@ -28,6 +28,7 @@ enum class Zones { // Keep sorted kGenAttentionQDotK, kGenAttentionSumHeads, kGenEmbed, + kGenEmbedding, kGenEmbeddingMatmul, kGenFFW, kGenImageTokens,