Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -1035,6 +1035,23 @@ cc_library(
],
)

cc_test(
name = "embeddings_test",
srcs = ["gemma/embeddings_test.cc"],
deps = [
":configs",
":gemma_args",
":gemma_lib",
":mat",
":matmul",
":threading_context",
":weights",
"@googletest//:gtest_main", # buildcleaner: keep
"//compression:types",
"@highway//:hwy",
],
)

cc_test(
name = "gemma_test",
srcs = [
Expand Down
1 change: 1 addition & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -376,6 +376,7 @@ set(GEMMA_TEST_FILES
compression/q4_0_test.cc
compression/sfp_test.cc
deepseek/deepseek_test.cc
gemma/embeddings_test.cc
gemma/gemma_args_test.cc
gemma/tensor_info_test.cc
gemma/weights_test.cc
Expand Down
14 changes: 14 additions & 0 deletions gemma/activations.h
Original file line number Diff line number Diff line change
Expand Up @@ -529,6 +529,12 @@ struct Activations {
s_ffw_in(num_layers, max_workers),
s_ffw_hidden(num_layers, max_workers),
s_ffw_out(num_layers, max_workers),
pooled(MatFactory("pooled", config.IsEmbedding() ? 1 : 0,
config.model_dim, ctx.allocator,
MatPadding::kPacked)),
h1(MatFactory("h1", config.IsEmbedding() ? 1 : 0,
config.EmbeddingProjDim(), ctx.allocator,
MatPadding::kPacked)),
router_in(MatFactory("router_in",
MoEBatchSize(moe_layer_config, batch_size),
config.model_dim, ctx.allocator)),
Expand Down Expand Up @@ -632,6 +638,10 @@ struct Activations {
if (config.hc_mult > 1) {
hc_mixes.AllocateAndAttachRowPtrs(row_ptrs);
}
if (config.IsEmbedding()) {
pooled.AllocateAndAttachRowPtrs(row_ptrs);
h1.AllocateAndAttachRowPtrs(row_ptrs);
}

if (config.is_encoder_decoder) {
t5_encoder_q.AllocateAndAttachRowPtrs(row_ptrs);
Expand Down Expand Up @@ -798,6 +808,10 @@ struct Activations {
TensorStats s_ffw_hidden; // after Activation+gating
TensorStats s_ffw_out;

// Embedding pooling and projection activations.
MatStorageT<float> pooled;
MatStorageT<float> h1;

// For MoE layers. These are used outside the expert-parallel loop:
MatStorageT<float> router_in;
MatStorageT<float> router_logits; // batch_size x num_experts
Expand Down
49 changes: 49 additions & 0 deletions gemma/configs.cc
Original file line number Diff line number Diff line change
Expand Up @@ -670,6 +670,48 @@ static ModelConfig ConfigGemma4_E4B() {
return config;
}

static LayerConfig LayerConfigEmbeddingGemma2_LM(size_t model_dim) {
LayerConfig config;
config.model_dim = model_dim;
config.ff_hidden_dim = 2048;
config.heads = 4;
config.kv_heads = 2;
config.qkv_dim = 256;
config.post_norm = PostNormType::Scale;
config.use_qk_norm = true;
config.norm_v = true;
config.ple_dim = 512;
return config;
}

static ModelConfig ConfigEmbeddingGemma2_LM() {
ModelConfig config = ConfigBaseGemmaV4();
config.display_name = "EmbeddingGemma2_LM";
config.model = Model::EMBEDDING_GEMMA2_LM;
config.wrapping = PromptWrapping::GEMMA_IT;
config.model_dim = 512;
config.vocab_size = kGemmaV3VocabSize;
// MatMul row limit (model supports 8192).
config.max_seq_len = kMaxBatchSize;
config.ple_dim = 512;
config.num_layers = 24;
config.use_global_timescale = true;
config.query_scale = QueryScaleType::One;
config.secondary_eos_id = 1;
LayerConfig layer_config = LayerConfigEmbeddingGemma2_LM(config.model_dim);
config.layer_configs = {config.num_layers, layer_config};
for (size_t i = 0; i < config.num_layers; ++i) {
if (i % 6 == 5) {
config.layer_configs[i].kv_heads = 1;
config.layer_configs[i].qkv_dim = 512;
}
}
// W = 513 gives [q - 512, q + 512] (|q - k| <= 512).
config.attention_window_sizes = RepeatedAttentionWindowSizes<24, 6>(
{513, 513, 513, 513, 513, config.max_seq_len});
return config;
}

static LayerConfig LayerConfigDeepSeek4_Flash(size_t model_dim) {
LayerConfig config;
config.model_dim = model_dim;
Expand Down Expand Up @@ -975,6 +1017,8 @@ static ModelConfig ConfigFromModel(Model model) {
return ConfigGemma4_E4B();
case Model::GEMMA4_E4B_LM:
return ConfigGemma4_E4B_LM();
case Model::EMBEDDING_GEMMA2_LM:
return ConfigEmbeddingGemma2_LM();
default:
HWY_ABORT("Model type %d unknown.", static_cast<int>(model));
}
Expand Down Expand Up @@ -1034,6 +1078,8 @@ const char* ModelPrefix(Model model) {
return "gemma4-e4b";
case Model::GEMMA4_E4B_LM:
return "gemma4-e4b-lm";
case Model::EMBEDDING_GEMMA2_LM:
return "embeddinggemma2-lm";
default:
HWY_ABORT("Model type %d unknown.", static_cast<int>(model));
}
Expand Down Expand Up @@ -1233,6 +1279,9 @@ Model DeduceModel(const Path& blob_path, size_t layers, int layer_types) {
case 18:
return Model::GEMMA3_270M;

case 24:
return Model::EMBEDDING_GEMMA2_LM;

case 26:
if (layer_types & (kDeducedViT | kDeducedKqNorm)) {
return Model::GEMMA3_1B;
Expand Down
20 changes: 19 additions & 1 deletion gemma/configs.h
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ HWY_INLINE_VAR constexpr int kSkipKV = 1;

HWY_INLINE_VAR constexpr size_t kMaxQKVDim = 1024;

constexpr size_t kEmbeddingGemma2Dim = 768;

#ifndef GEMMA_FUSED_FFN
#define GEMMA_FUSED_FFN 1
#endif // !GEMMA_FUSED_FFN
Expand Down Expand Up @@ -328,6 +330,7 @@ enum class Model {
GEMMA4_E2B_LM = GEMMA4_2B_LM,
GEMMA4_E4B,
GEMMA4_E4B_LM,
EMBEDDING_GEMMA2_LM,
kSentinel,
};

Expand All @@ -349,6 +352,10 @@ static inline bool IsQwen3(Model model) {
model == Model::QWEN3_4B;
}

static inline bool IsEmbeddingGemma2(Model model) {
return model == Model::EMBEDDING_GEMMA2_LM;
}

static inline bool IsObsolete(Model model) {
const size_t i = static_cast<size_t>(model);
if (i == 5 || i == 6 || i == 8 || i == 9) return true;
Expand Down Expand Up @@ -849,7 +856,18 @@ struct ModelConfig : public IFields {
}

bool IsEOS(int id) const { return (id == eos_id || id == secondary_eos_id); }
bool IsEmbedding() const { return false; }

bool IsEmbedding() const {
if (IsEmbeddingGemma2(model)) return true;
return false;
}

size_t EmbeddingProjDim() const {
if (IsEmbeddingGemma2(model)) return kEmbeddingGemma2Dim;
return 0;
}

bool HasPLETable() const { return ple_dim > 0 && !IsEmbeddingGemma2(model); }

// Major version of the model family, reflecting architecture changes. This is
// more convenient to compare than `Model` because that also includes the
Expand Down
49 changes: 49 additions & 0 deletions gemma/configs_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -239,4 +239,53 @@ TEST(ConfigsTest, Gemma4E4BQuantizerSpecifiers) {
EXPECT_EQ(e4b_lm_pt.wrapping, PromptWrapping::GEMMA_PT);
}

TEST(ConfigsTest, EmbeddingGemma2) {
const ModelConfig config(Model::EMBEDDING_GEMMA2_LM, Type::kBF16,
PromptWrapping::GEMMA_IT);
EXPECT_EQ(config.display_name, "EmbeddingGemma2_LM");
EXPECT_FALSE(config.HasGemma4Vit());
EXPECT_TRUE(IsEmbeddingGemma2(config.model));
EXPECT_TRUE(config.IsEmbedding());
EXPECT_EQ(config.EmbeddingProjDim(), kEmbeddingGemma2Dim);
EXPECT_EQ(config.num_layers, 24u);
EXPECT_EQ(config.model_dim, 512u);
EXPECT_EQ(config.ple_dim, 512u);
EXPECT_FALSE(config.HasPLETable());
EXPECT_EQ(config.vocab_size, 262144u);
EXPECT_EQ(config.max_seq_len, kMaxBatchSize);
EXPECT_EQ(config.query_scale, QueryScaleType::One);
EXPECT_TRUE(config.use_global_timescale);
EXPECT_FLOAT_EQ(config.partial_rotary_factor, 1.0f);
EXPECT_EQ(config.NumPrefillLayers(), 24u);

for (size_t i = 0; i < 24; ++i) {
const LayerConfig& lc = config.layer_configs[i];
EXPECT_EQ(lc.ple_dim, 512u) << i;
EXPECT_EQ(lc.ff_hidden_dim, 2048u) << i;
EXPECT_EQ(lc.heads, 4u) << i;
EXPECT_EQ(lc.post_qk, PostQKType::Rope) << i;
EXPECT_TRUE(lc.use_qk_norm) << i;
EXPECT_TRUE(lc.norm_v) << i;
const bool is_global = (i % 6 == 5);
EXPECT_EQ(lc.kv_heads, is_global ? 1u : 2u) << i;
EXPECT_EQ(lc.qkv_dim, is_global ? 512u : 256u) << i;
EXPECT_EQ(config.attention_window_sizes[i],
is_global ? config.max_seq_len : 513u)
<< i;
}

const ModelConfig bf16_it("embeddinggemma2-lm-bf16-it");
EXPECT_EQ(bf16_it.model, Model::EMBEDDING_GEMMA2_LM);
EXPECT_EQ(bf16_it.weight, Type::kBF16);
EXPECT_EQ(bf16_it.wrapping, PromptWrapping::GEMMA_IT);

const ModelConfig sfp_pt("embeddinggemma2-lm-sfp-pt");
EXPECT_EQ(sfp_pt.model, Model::EMBEDDING_GEMMA2_LM);
EXPECT_EQ(sfp_pt.weight, Type::kSFP);
EXPECT_EQ(sfp_pt.wrapping, PromptWrapping::GEMMA_PT);

EXPECT_EQ(DeduceModel(Path("embeddinggemma2-lm-bf16-it.sbs"), 24, 0),
Model::EMBEDDING_GEMMA2_LM);
}

} // namespace gcpp
Loading
Loading