Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -1056,7 +1056,6 @@ cc_test(
cc_test(
name = "model_health_test",
srcs = ["evals/model_health_test.cc"],
linkstatic = True,
# Requires model files
tags = [
"local",
Expand All @@ -1068,7 +1067,8 @@ cc_test(
":configs",
":cross_entropy",
":gemma_lib",
"@googletest//:gtest_main", # buildcleaner: keep
":test_util",
"//testing/base/public:gunit_for_library_testonly", # buildcleaner: keep
"@highway//:hwy",
"@highway//:hwy_test_util",
],
Expand All @@ -1077,7 +1077,6 @@ cc_test(
cc_test(
name = "gemma_batch_bench",
srcs = ["evals/gemma_batch_bench.cc"],
linkstatic = True,
# Requires model files
tags = [
"local",
Expand All @@ -1088,7 +1087,7 @@ cc_test(
":benchmark_helper",
":gemma_lib",
":test_util",
"@googletest//:gtest_main", # buildcleaner: keep
"//testing/base/public:gunit_for_library_testonly", # buildcleaner: keep
"@highway//:hwy",
"@highway//:hwy_test_util",
"@highway//:nanobenchmark",
Expand Down
1 change: 1 addition & 0 deletions compression/compress.cc
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

#include "compression/compress.h"

#include <cmath>
#include <stddef.h>
#include <stdint.h>

Expand Down
6 changes: 4 additions & 2 deletions compression/python/compression_clif_aux.cc
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,10 @@ class SbsWriterImpl : public ISbsWriter {
}

HWY_ASSERT(weights.size() == mat.Extents().Area());
Compress(weights.data(), weights.size(), working_set_, mat.Span(),
/*packed_ofs=*/0, ctx_);
{
Compress(weights.data(), weights.size(), working_set_, mat.Span(),
/*packed_ofs=*/0, ctx_);
}
writer_.Add(name, mat.Packed(), mat.PackedBytes());
}

Expand Down
7 changes: 6 additions & 1 deletion compression/python/compression_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,10 @@ def test_sbs_writer(self):
configs.Type.kBF16,
info_256,
)
f32_data = np.array([0.000375] * 128 + [0.00006] * 128, dtype=np.float32)
writer.insert(
"tensor_f32",
np.array([0.000375] * 128 + [0.00006] * 128, dtype=np.float32),
f32_data,
configs.Type.kF32,
info_256,
)
Expand All @@ -105,6 +106,10 @@ def test_sbs_writer(self):
tokenizer_blob = "" # no tokenizer required for testing
writer.write(config, tokenizer_blob)

with open(temp_file.full_path, "rb") as f:
sbs_bytes = f.read()
self.assertIn(f32_data.tobytes(), sbs_bytes)

print("Ignore next two warnings; test does not enable model deduction.")
reader = compression.SbsReader(temp_file.full_path)

Expand Down
19 changes: 13 additions & 6 deletions compression/types.h
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,8 @@ constexpr bool IsMxFp4Stream() {
template <typename Packed>
constexpr bool IsPacked() {
return IsNuqStream<Packed>() || IsI8Stream<Packed>() ||
IsQ4_0Stream<Packed>() || IsMxFp4Stream<Packed>();
IsQ4_0Stream<Packed>() || IsMxFp4Stream<Packed>()
;
}

template <typename Packed>
Expand All @@ -294,12 +295,14 @@ enum class Type {
kInt8,
kQ4_0,
kMXFP4,
kReserved14,
};
// These are used in `ModelConfig.Specifier`, hence the strings will not
// change, though new ones may be added.
static constexpr const char* kTypeStrings[] = {
"unknown", "f32", "bf16", "sfp", "nuq", "f64", "u32",
"u64", "i8", "u16", "u8", "int8", "q4_0", "mxfp4"};
"unknown", "f32", "bf16", "sfp", "nuq", "f64", "u32", "u64",
"i8", "u16", "u8", "int8", "q4_0", "mxfp4", "reserved14"
};
static constexpr size_t kNumTypes =
sizeof(kTypeStrings) / sizeof(kTypeStrings[0]);
static constexpr size_t kTypeBits[] = {
Expand All @@ -317,6 +320,7 @@ static constexpr size_t kTypeBits[] = {
8 * sizeof(int8_t),
4 /* Q4_0Stream, actually 4.5 */,
4 /* MxFp4Stream, actually 4.25 */,
0 /* reserved */,
};

static inline bool EnumValid(Type type) {
Expand Down Expand Up @@ -376,17 +380,20 @@ constexpr bool IsCompressed() {
hwy::IsSame<hwy::RemoveCvRef<Packed>, NuqStream>() ||
hwy::IsSame<hwy::RemoveCvRef<Packed>, I8Stream>() ||
hwy::IsSame<hwy::RemoveCvRef<Packed>, Q4_0Stream>() ||
hwy::IsSame<hwy::RemoveCvRef<Packed>, MxFp4Stream>();
hwy::IsSame<hwy::RemoveCvRef<Packed>, MxFp4Stream>()
;
}

static inline bool IsCompressed(Type type) {
return type == Type::kSFP || type == Type::kNUQ || type == Type::kI8 ||
type == Type::kQ4_0 || type == Type::kMXFP4;
type == Type::kQ4_0 || type == Type::kMXFP4
;
}

static inline bool IsPacked(Type type) {
return type == Type::kNUQ || type == Type::kI8 || type == Type::kQ4_0 ||
type == Type::kMXFP4;
type == Type::kMXFP4
;
}

static inline bool SupportsPointerArithmetic(Type type) {
Expand Down
3 changes: 2 additions & 1 deletion evals/model_health_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
#include "evals/cross_entropy.h"
#include "gemma/configs.h"
#include "gemma/gemma.h"
#include "util/test_util.h"
#include "hwy/base.h"
#include "hwy/tests/hwy_gtest.h"

Expand Down Expand Up @@ -485,7 +486,7 @@ TEST_F(ModelHealthTest, DeterministicGeneration) {
} // namespace gcpp

int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
gcpp::InternalInitTest();
gcpp::ModelHealthTest::InitEnv(argc, argv);
int ret = RUN_ALL_TESTS();
gcpp::ModelHealthTest::DeleteEnv();
Expand Down
3 changes: 2 additions & 1 deletion gemma/activations.h
Original file line number Diff line number Diff line change
Expand Up @@ -827,7 +827,8 @@ struct Activations {
config.model_dim, allocator)),
mla_o_in(MatFactory("mla_o_in",
mla_dims.o_in_dim > 0 ? batch_size : 0,
mla_dims.o_in_dim, allocator)) {
mla_dims.o_in_dim, allocator))
{
moe_C1.AllocateAndAttachRowPtrs(row_ptrs);
moe_C2.AllocateAndAttachRowPtrs(row_ptrs);
ffw_expert_in.AllocateAndAttachRowPtrs(row_ptrs);
Expand Down
2 changes: 1 addition & 1 deletion gemma/configs.cc
Original file line number Diff line number Diff line change
Expand Up @@ -481,7 +481,7 @@ static LayerConfig LayerConfigGemma4_26B_MoE_LM(size_t model_dim) {
static ModelConfig ConfigGemma4_26B_MoE() {
ModelConfig config = ConfigBaseGemmaV4();
config.display_name = "Gemma4_26B_MoE";
config.final_cap = 0.0f;
config.final_cap = 30.0f;
config.att_cap = 0.0f;
config.model = Model::GEMMA4_26B_MOE;
config.wrapping = PromptWrapping::GEMMA_IT;
Expand Down
10 changes: 10 additions & 0 deletions gemma/configs_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -214,4 +214,14 @@ TEST(ConfigsTest, Gemma4E4BQuantizerSpecifiers) {
EXPECT_EQ(e4b_lm_pt.wrapping, PromptWrapping::GEMMA_PT);
}

TEST(ConfigsTest, Gemma4MoEConfig) {
const ModelConfig moe(Model::GEMMA4_26B_MOE, Type::kSFP,
PromptWrapping::GEMMA_IT);
EXPECT_EQ(moe.display_name, "Gemma4_26B_MoE");
EXPECT_FLOAT_EQ(moe.final_cap, 30.0f);
EXPECT_FLOAT_EQ(moe.att_cap, 0.0f);
EXPECT_EQ(moe.model_dim, 2816u);
EXPECT_EQ(moe.num_layers, 30u);
}

} // namespace gcpp
45 changes: 26 additions & 19 deletions gemma/gemma4_moe.cc
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
#include "gemma/attention.h" // includes highway.h
#include "gemma/tiled_attention.h"
#include "gemma/gemma-inl.h"
#include "ops/fast_ops-inl.h"
#include "ops/ops-inl.h"

HWY_BEFORE_NAMESPACE();
Expand Down Expand Up @@ -239,6 +240,8 @@ struct Gemma4MoE {
activations.s_expert_in.Notify(layer.layer_idx, tmp_in, env.ctx, 0,
cluster_idx, parallelism);

const size_t expert_ff_hidden_dim =
layer.moe_gating_einsum_w1[expert_idx].Rows();
const ActivationType activation = layer.layer_config.activation;
MatPtrT<BF16>& C1 = per_cluster.moe_C1;
C1.OverrideRows(expert_size);
Expand Down Expand Up @@ -275,19 +278,20 @@ struct Gemma4MoE {
cluster_idx, parallelism);

// Hidden layer -> output layer.
size_t expert_ff_hidden_dim = layer.moe_gating_einsum_w1[expert_idx].Rows();
MatStorageT<BF16> C1_narrow("C1_n",
Extents2D(expert_size, expert_ff_hidden_dim),
env.ctx.allocator, MatPadding::kOdd);
for (size_t i = 0; i < expert_size; ++i) {
memcpy(C1_narrow.Row(i), C1.Row(i), expert_ff_hidden_dim * sizeof(BF16));
{
MatStorageT<BF16> C1_narrow("C1_n",
Extents2D(expert_size, expert_ff_hidden_dim),
env.ctx.allocator, MatPadding::kOdd);
for (size_t i = 0; i < expert_size; ++i) {
memcpy(C1_narrow.Row(i), C1.Row(i),
expert_ff_hidden_dim * sizeof(BF16));
}
CallMatMul(C1_narrow, layer.moe_linear_w[expert_idx],
/*add=*/nullptr, env, expert_out, options);
}


CallMatMul(C1_narrow, layer.moe_linear_w[expert_idx],
/*add=*/nullptr, env, expert_out, options);

if (layer.p_expert_sc.HasPtr()) {
if (layer.p_expert_sc.HasPtr()
) {
const MatPtrT<BF16> sc_mat(layer.p_expert_sc);
const BF16* sc_data = sc_mat.Row(0);
const float expert_scale =
Expand Down Expand Up @@ -466,9 +470,12 @@ void Gemma4MoETransformerLayer(size_t num_tokens, size_t layer_idx,
/*is_attention=*/true, env.ctx);

// Dual-Path FFW
pre_norm(
layer.pre_ffw2_ns.HasPtr() ? layer.pre_ffw2_ns : layer.pre_ffw_norm_scale,
activations.pre_ffw_rms_out);
const MatPtr& shared_norm = layer.pre_ffw2_ns.HasPtr() ? layer.pre_ffw2_ns : layer.pre_ffw_norm_scale;
const MatPtr& moe_norm = layer.pre_ffw_norm_scale;
const MatPtr& shared_post_norm = layer.post_ffw2_ns;
const MatPtr& moe_post_norm = layer.post_ffw1_ns;

pre_norm(shared_norm, activations.pre_ffw_rms_out);

// Shared MLP Path
FFWNoVit(layer, activations, env); // writes to activations.ffw_out
Expand All @@ -482,17 +489,17 @@ void Gemma4MoETransformerLayer(size_t num_tokens, size_t layer_idx,
}
}

if (layer.post_ffw2_ns.HasPtr()) {
rms_norm_inplace(layer.post_ffw2_ns, activations.attention.att_sums);
if (shared_post_norm.HasPtr()) {
rms_norm_inplace(shared_post_norm, activations.attention.att_sums);
}

// MoE Path
pre_norm(layer.pre_ffw_norm_scale, activations.pre_ffw_rms_out);
pre_norm(moe_norm, activations.pre_ffw_rms_out);

Gemma4MoE::MoEFFW(layer, activations, env); // writes to activations.ffw_out

if (layer.post_ffw1_ns.HasPtr()) {
rms_norm_inplace(layer.post_ffw1_ns, activations.ffw_out);
if (moe_post_norm.HasPtr()) {
rms_norm_inplace(moe_post_norm, activations.ffw_out);
}

// Combine & Final Norm (Fix for dual-path combination)
Expand Down
3 changes: 3 additions & 0 deletions gemma/tensor_info.h
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ struct TensorInfo {
// If false, then [10, 20, 30] -> [10*20, 30] and [30] -> [1, 30].
// If true, then [10, 20, 30] -> [10, 20*30] and [30] -> [1, 30].
bool cols_take_extra_dims = false;
// Optional pre-computed scale (e.g. for kW2_UL weights from QAFT).
// If > 0.0, used directly instead of re-estimating via ScaleWeightsW2UL.
float scale = 0.0f;
};

// Collapses/expands the tensor dims into 2D extents, which may be 0, 0 for
Expand Down
19 changes: 17 additions & 2 deletions gemma/weights.cc
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include <stdio.h>
#include <stdlib.h>

#include <cstring>
#include <mutex> // NOLINT
#include <string>
#include <vector>
Expand Down Expand Up @@ -129,7 +130,10 @@ static void SplitPackedMatrix(MatPtr& parent, size_t split_row, MatPtr& w1,
void LayerWeightsPtrs::SplitW1() {
// Used for Gemma layers; FFWVit uses different tensors.
if (layer_config.type == LayerAttentionType::kVit) return;
if (layer_config.IsMoE()) return;
if (layer_config.IsMoE() && !gating_einsum_w.HasPtr() &&
!gating_einsum_w1.HasPtr()) {
return;
}

// Files have both or neither of w1 and w2.
HWY_ASSERT(gating_einsum_w1.HasPtr() == gating_einsum_w2.HasPtr());
Expand Down Expand Up @@ -543,6 +547,18 @@ void LayerWeightsPtrs::Fixup(Model model, std::vector<MatOwner>& mat_owners,
const size_t elem_bytes = qkv_einsum_w2.ElementBytes();
const size_t old_row_bytes = old_stride * elem_bytes;
const size_t kv_heads = layer_config.kv_heads;
const size_t qkv_dim = layer_config.qkv_dim;

// In Gemma 4 global layers, attention_k_eq_v is true (K0 == V0).
// If already interleaved by exporter: [K0, V0, K1, V1], slice 0 == slice 1.
// If not interleaved: [K0, K1, V0, V1], slice 0 (K0) != slice 1 (K1).
const uint8_t* slice0 = qkv_einsum_w2.RowBytes(0);
const uint8_t* slice1 = qkv_einsum_w2.RowBytes(qkv_dim);
if (std::memcmp(slice0, slice1, old_row_bytes) == 0) {
// Exporter already emitted interleaved layout; skip fixup.
return;
}

const size_t total_bytes = qkv_einsum_w2.Rows() * old_row_bytes;
hwy::AlignedFreeUniquePtr<uint8_t[]> tmp =
hwy::AllocateAligned<uint8_t>(total_bytes);
Expand All @@ -556,7 +572,6 @@ void LayerWeightsPtrs::Fixup(Model model, std::vector<MatOwner>& mat_owners,
}

const size_t new_row_bytes = qkv_einsum_w2.Cols() * elem_bytes;
const size_t qkv_dim = layer_config.qkv_dim;
const uint8_t* src_ptr = tmp.get();
for (size_t i = 0; i < kv_heads; ++i) {
for (size_t row = 0; row < qkv_dim; ++row) {
Expand Down
Loading
Loading