From 49809b72b474a4f795be2d7e436f0bfd712f1d3e Mon Sep 17 00:00:00 2001 From: Tushar Labhane Date: Fri, 3 Apr 2026 20:08:23 +0530 Subject: [PATCH] feat: add Gemma 4 model support (E2B, E4B, 26B A4B, 31B) --- gemma/activations.h | 302 ++++++--------------------------------- gemma/attention.cc | 310 +++++++++++++++------------------------- gemma/configs.cc | 262 ++++++++++++++++++++++++++++------ gemma/configs.h | 123 +++++++--------- gemma/gemma-inl.h | 166 +++++++++++++++++----- gemma/gemma.cc | 329 +++++++++---------------------------------- gemma/tensor_info.cc | 62 ++++++-- gemma/weights.h | 28 +++- 8 files changed, 693 insertions(+), 889 deletions(-) diff --git a/gemma/activations.h b/gemma/activations.h index c14b24e2..6b2aacd2 100644 --- a/gemma/activations.h +++ b/gemma/activations.h @@ -23,97 +23,66 @@ #include #include -#include "gemma/configs.h" // ModelConfig -#include "gemma/flash_structs.h" -#include "gemma/gemma_args.h" // AttentionImpl -#include "gemma/kv_cache.h" -#include "gemma/tensor_stats.h" -#include "ops/ops.h" // CreateInvTimescale -#include "util/basics.h" // BF16 -#include "util/mat.h" // MatStorageT +#include "gemma/configs.h" // ModelConfig +#include "ops/ops.h" // CreateInvTimescale +#include "util/basics.h" // BF16 +#include "util/mat.h" // MatStorageT #include "util/threading_context.h" namespace gcpp { -typedef std::vector> AlignedFloatVector; -typedef std::vector> AlignedBF16Vector; - -// Returns the scale value to use for the query in the attention computation. -// Also called by ops_test. -static inline float ChooseQueryScale(const ModelConfig& config) { - const LayerConfig& layer_config = config.layer_configs[0]; - if (config.query_scale == QueryScaleType::SqrtModelDimDivNumHeads) - return 1.0f / - sqrtf(static_cast(config.model_dim / layer_config.heads)); - // QueryScaleType::SqrtKeySize - return 1.0f / sqrtf(static_cast(layer_config.qkv_dim)); -} - struct AttentionActivations { + // Returns the scale value to use for the query in the attention computation. + // Also called by ops_test. + static inline float ChooseQueryScale(const ModelConfig& config) { + const LayerConfig& layer_config = config.layer_configs[0]; + if (config.query_scale == QueryScaleType::SqrtModelDimDivNumHeads) + return 1.0f / + sqrtf(static_cast(config.model_dim / layer_config.heads)); + // QueryScaleType::SqrtKeySize + return 1.0f / sqrtf(static_cast(layer_config.qkv_dim)); + } + AttentionActivations( const ModelConfig& config, const LayerConfig& layer_config, - size_t batch_size, size_t seq_len, const RuntimeConfig& runtime_config, - size_t max_workers, const Allocator& allocator, + size_t batch_size, size_t seq_len, const Allocator& allocator, std::vector>& row_ptrs) - : rep_factor(max_workers * - AttentionActivations::kThreadReplicationFactor / - layer_config.heads), - // `vocab_size == 0` means it is for Vit part, VitAttention - // is still MHA and does not use an external KV cache. + : config(config), + + // `vocab_size == 0` means it is for Vit part, VitAttention is still MHA + // and does not use an external KV cache. q(MatFactory("q", batch_size, config.vocab_size == 0 ? layer_config.heads * 3 * layer_config.qkv_dim : layer_config.heads * layer_config.qkv_dim, allocator)), - q_bf(MatFactory("q_bf", batch_size, - config.vocab_size == 0 - ? layer_config.heads * 3 * layer_config.qkv_dim - : layer_config.heads * layer_config.qkv_dim, - allocator)), q_T(MatFactory("q_T", layer_config.qkv_dim, config.vocab_size == 0 ? batch_size * layer_config.heads * 3 : batch_size * layer_config.heads, allocator)), - vit_Q(MatFactory("Q2", batch_size, layer_config.qkv_dim, allocator)), - vit_K_T(MatFactory( - "K2_T", hwy::RoundUpTo(seq_len, kMaxBF16PerVector), - layer_config.heads * - hwy::RoundUpTo(layer_config.qkv_dim, kMaxBF16PerVector), - allocator, MatPadding::kPacked)), - vit_V_T(MatFactory( - "V2_T", hwy::RoundUpTo(seq_len, kMaxBF16PerVector), - layer_config.heads * - hwy::RoundUpTo(layer_config.qkv_dim, kMaxBF16PerVector), - allocator, MatPadding::kPacked)), pre_att_rms_out(MatFactory("pre_att_rms_out", batch_size, config.model_dim, allocator)), - // att is only valid for AttentionImpl::kOld. - att(MatFactory( - "att", batch_size, - layer_config.heads * - (runtime_config.attention_impl == AttentionImpl::kOld ? seq_len - : 1), - allocator)), + att(MatFactory("att", batch_size, layer_config.heads * seq_len, + allocator)), att_out(MatFactory("att_out", batch_size, layer_config.heads * layer_config.qkv_dim, allocator)), - att_out_reps(MatFactory("att_out", batch_size * rep_factor, - layer_config.heads * layer_config.qkv_dim, - allocator)), - softmax_max(MatFactory("softmax_max", batch_size, layer_config.heads, - allocator)), - softmax_d( - MatFactory("softmax_d", batch_size, layer_config.heads, allocator)), att_sums( MatFactory("att_sums", batch_size, config.model_dim, allocator)), inv_timescale( CreateInvTimescale(allocator, layer_config.qkv_dim, - layer_config.post_qk == PostQKType::HalfRope)), + layer_config.post_qk == PostQKType::HalfRope, + config.rope_theta_sliding)), inv_timescale_global(CreateInvTimescale( allocator, layer_config.qkv_dim, - layer_config.post_qk == PostQKType::HalfRope, 1000000.0)) { + layer_config.post_qk == PostQKType::HalfRope, + config.rope_theta_full)), + + div_seq_len(static_cast(seq_len)), + div_heads(static_cast(layer_config.heads)), + query_scale(ChooseQueryScale(config)) { // Batch size can be 0 in experimental code so do not assert. if (batch_size == 0) { static std::atomic_flag warned = ATOMIC_FLAG_INIT; @@ -122,204 +91,49 @@ struct AttentionActivations { } return; } - // This is a guess at the maximum number of params we might need to avoid - // reallocations. The actual number of params is determined by the number of - // query tiles, which is not known here. - flash_params.reserve(batch_size * layer_config.heads); - split_flash_params.reserve(batch_size * layer_config.heads); // For MatMul outputs, precompute their row pointers. // If we forget any MatMul outputs here, debug builds print a warning but // fill them in each MatMul call. q.AllocateAndAttachRowPtrs(row_ptrs); - q_bf.AllocateAndAttachRowPtrs(row_ptrs); q_T.AllocateAndAttachRowPtrs(row_ptrs); att_sums.AllocateAndAttachRowPtrs(row_ptrs); } void SetBatchSize(size_t batch_size) { q.OverrideRows(batch_size); - q_bf.OverrideRows(batch_size); // q_T rows are always qkv_dim! - vit_Q.OverrideRows(batch_size); - // vit_K_T and vit_V_T stay seq_len! - pre_att_rms_out.OverrideRows(batch_size); att.OverrideRows(batch_size); att_out.OverrideRows(batch_size); - att_out_reps.OverrideRows(batch_size * rep_factor); - // There is no override for [split_]flash_params, because we reserved an - // upper bound, and flash attention controls the actual size when it - // calculates the size and number of tiles. - softmax_max.OverrideRows(batch_size); - softmax_d.OverrideRows(batch_size); att_sums.OverrideRows(batch_size); - - // `inv_timescale*` are not batched. } - // Maximum factor by which we might scale-up work to maximize parallelism. - size_t rep_factor = 1; - // Parameters for flash attention. The size of the vector is somewhere between - // the number of query rows and 1/8th of that. - std::vector flash_params; - // Parameters for flash attention, split by k-position. May be significantly - // larger than flash_params in decode mode, when the number of query rows is - // small. - std::vector split_flash_params; - MatStorageT q; // query - MatStorageT q_bf; - MatStorageT q_T; // Transposed to maximize attention speed. + const ModelConfig& config; - MatStorageT vit_Q; - MatStorageT vit_K_T; - MatStorageT vit_V_T; + MatStorageT q; // query + MatStorageT q_T; // Transposed to maximize attention speed. MatStorageT pre_att_rms_out; - MatStorageT att; // attention vector - MatStorageT att_out; // attention output - MatStorageT att_out_reps; // attention output for each thread. - MatStorageT softmax_max; // see OnlineSoftmaxState - MatStorageT softmax_d; // see OnlineSoftmaxState + MatStorageT att; // attention vector + MatStorageT att_out; // attention output // Accumulation of attention outputs over heads MatStorageT att_sums; - MatStorageT k_tile_vec; - MatStorageT v_tile_vec; - std::vector> sub_task_att_out; - std::vector - sub_task_exp_denominator_sums; - std::vector - sub_task_max_logits; - // Rope MatStorageT inv_timescale; MatStorageT inv_timescale_global; - // Replication factor to help evenly share work over threads. - static constexpr size_t kThreadReplicationFactor = 4; -}; - -// A non-owning view of AttentionActivations. -struct AttentionActivationsPtrs { - AttentionActivationsPtrs(const ModelConfig& config, size_t seq_len, - std::vector& flash_params, - std::vector& split_flash_params) - : config(config), - flash_params(flash_params), - split_flash_params(split_flash_params), - div_seq_len(static_cast(seq_len)), - div_heads(static_cast(config.layer_configs[0].heads)), - query_scale(ChooseQueryScale(config)) {} - - AttentionActivationsPtrs(const ModelConfig& config, size_t seq_len, - AttentionActivations& activations) - : AttentionActivationsPtrs(config, seq_len, activations.flash_params, - activations.split_flash_params) { - q = activations.q; - q_bf = activations.q_bf; - q_T = activations.q_T; - vit_Q = activations.vit_Q; - vit_K_T = activations.vit_K_T; - vit_V_T = activations.vit_V_T; - pre_att_rms_out = activations.pre_att_rms_out; - att = activations.att; - att_out = activations.att_out; - att_out_reps = activations.att_out_reps; - softmax_max = activations.softmax_max; - softmax_d = activations.softmax_d; - att_sums = activations.att_sums; - inv_timescale = activations.inv_timescale; - inv_timescale_global = activations.inv_timescale_global; - } - void SetBatchSize(size_t batch_size) { - q.OverrideRows(batch_size); - q_bf.OverrideRows(batch_size); - // q_T rows are always qkv_dim! - - vit_Q.OverrideRows(batch_size); - // vit_K_T and vit_V_T stay seq_len! - - pre_att_rms_out.OverrideRows(batch_size); - att.OverrideRows(batch_size); - att_out.OverrideRows(batch_size); - softmax_max.OverrideRows(batch_size); - softmax_d.OverrideRows(batch_size); - att_sums.OverrideRows(batch_size); - // `inv_timescale*` are not batched. - } - - size_t SeqLen() const { - return static_cast(div_seq_len.GetDivisor()); - } - - const ModelConfig& config; - // Parameters for flash attention. - std::vector& flash_params; - std::vector& split_flash_params; - - // For the matrices below, the batch_size dimension is really qbatch.Size() * - // token_batch_size, but in all known uses, one of those is 1. Specifically, - // during PrefillTBatch, it is prompt length (up to some max batch size) - // and otherwise it's qbatch.Size(). - - // Query matrix of size batch_size x (q_heads * qkv_dim). - MatPtrT q; - // Query matrix of size batch_size x (q_heads * qkv_dim). - MatPtrT q_bf; - // Transposed query matrix for faster Q*K^T. - MatPtrT q_T; - - MatPtrT vit_Q; - MatPtrT vit_K_T; - MatPtrT vit_V_T; - - // Output of RMSNorm before attention, size batch_size x model_dim. - MatPtrT pre_att_rms_out; - // Attention scores computed from Q*K^T, size batch_size x (q_heads * - // seq_len). - MatPtrT att; - // Attention output computed from att * V, size batch_size x (q_heads * - // qkv_dim). - MatPtrT att_out; - MatPtrT att_out_reps; - // The maximum logit value encountered when computing att_out from att, - // size batch_size x q_heads . See OnlineSoftmaxState for details. - // WARNING: Only filled in for AttentionImpl::kOld. - MatPtrT softmax_max; - // The sum of scaled exponentials when computing att_out from att, - // size batch_size x q_heads . See OnlineSoftmaxState for details. - // WARNING: Only filled in for AttentionImpl::kOld. - MatPtrT softmax_d; - // Accumulation of attention outputs over heads, size batch_size x - // model_dim. - MatPtrT att_sums; - // Stores intermediate results of computing QKV, - // [qbatch * kv_heads , k_tile_size * qkv_dim] - MatPtrT k_tile_vec; - MatPtrT v_tile_vec; - // Used by TiledFlashAttention to store intermediate results. - std::vector>* sub_task_att_out; - std::vector* - sub_task_exp_denominator_sums; - std::vector* - sub_task_max_logits; - // Inverse timescales for RoPE computation. - MatPtrT inv_timescale; - // Inverse timescales for global RoPE computation. - MatPtrT inv_timescale_global; - // Divisor for faster division by sequence length. hwy::Divisor div_seq_len; - // Divisor for faster division by number of heads. + // Unfortunately, some models have had non-power-of-two heads. hwy::Divisor div_heads; - // Query scaling factor for attention computation. float query_scale; }; struct Activations { - Activations(const RuntimeConfig& runtime_config, const ModelConfig& config, - size_t batch_size, size_t seq_len, ThreadingContext& ctx, + Activations(const ModelConfig& config, size_t batch_size, size_t seq_len, + ThreadingContext& ctx, std::vector>& row_ptrs) : layer_config(config.layer_configs[0]), @@ -338,19 +152,8 @@ struct Activations { ffw_out( MatFactory("ffw_out", batch_size, config.model_dim, ctx.allocator)), - max_workers(ctx.pools.MaxWorkers()), - s_ffw_in(config.num_layers, max_workers), - s_ffw_hidden(config.num_layers, max_workers), - s_ffw_out(config.num_layers, max_workers), - - s_w_gating_einsum_w1(config.num_layers, max_workers), - s_w_gating_einsum_w2(config.num_layers, max_workers), - s_w_linear_w(config.num_layers, max_workers), - attention_impl(runtime_config.attention_impl), - attention_storage(config, layer_config, batch_size, seq_len, - runtime_config, ctx.pools.MaxWorkers(), ctx.allocator, - row_ptrs), - attention(config, seq_len, attention_storage) { + attention(config, layer_config, batch_size, seq_len, ctx.allocator, + row_ptrs) { HWY_ASSERT(batch_size != 0); // For MatMul outputs, precompute their row pointers. @@ -366,12 +169,6 @@ struct Activations { // Note that BindC on any MatMul output considerably slows down Prefill. } - ~Activations() { - s_ffw_in.ReduceAndPrint("ffw_in"); - s_ffw_hidden.ReduceAndPrint("ffw_hidden"); - s_ffw_out.ReduceAndPrint("ffw_out"); - } - // Negligible CPU time. void SetBatchSize(size_t batch_size) { x.OverrideRows(batch_size); @@ -384,15 +181,12 @@ struct Activations { C2.OverrideRows(batch_size); ffw_out.OverrideRows(batch_size); - attention_storage.SetBatchSize(batch_size); - // `AttentionActivationsPtrs` holds `MatPtrT` which also require updating; - // their row override is not updated when the underlying storage changes. attention.SetBatchSize(batch_size); } const LayerConfig& layer_config; - MatStorageT x; // input + MatStorageT x; // input MatStorageT x_bf; // output of final RMSNorm, input to EmbeddingMatmul MatStorageT logits; // TODO: BF16 after Softmax supports that. MatStorageT sampled; // batch_size x 3 (padded) @@ -403,19 +197,7 @@ struct Activations { MatStorageT C2; MatStorageT ffw_out; - const size_t max_workers; - TensorStats s_ffw_in; - TensorStats s_ffw_hidden; // after Activation+gating - TensorStats s_ffw_out; - - TensorStats s_w_gating_einsum_w1; - TensorStats s_w_gating_einsum_w2; - TensorStats s_w_linear_w; - - AttentionImpl attention_impl; - - AttentionActivations attention_storage; - AttentionActivationsPtrs attention; + AttentionActivations attention; }; } // namespace gcpp diff --git a/gemma/attention.cc b/gemma/attention.cc index 5aab57ec..5c7a34d6 100644 --- a/gemma/attention.cc +++ b/gemma/attention.cc @@ -20,13 +20,13 @@ #include "compression/types.h" // GEMMA_DISABLED_TARGETS #include "util/zones.h" -#include "hwy/base.h" #ifndef HWY_DISABLED_TARGETS #define HWY_DISABLED_TARGETS GEMMA_DISABLED_TARGETS #endif // HWY_DISABLED_TARGETS #include "gemma/activations.h" #include "gemma/configs.h" // kMaxQKVDim +#include "gemma/gemma.h" #include "gemma/weights.h" #include "util/threading.h" #include "util/threading_context.h" @@ -43,83 +43,12 @@ // After highway.h #include "compression/compress-inl.h" #include "gemma/flash_attention.h" -#include "gemma/gemma-inl.h" #include "ops/ops-inl.h" HWY_BEFORE_NAMESPACE(); namespace gcpp { namespace HWY_NAMESPACE { -// Returns the number of floats per vector (aka NF). -size_t FloatsPerVector() { - using DF = hn::ScalableTag; - const DF df; - return hn::Lanes(df); -} - -// The k-cache and v-cache are setup without knowing NF. So if it hasn't been -// done already, reshape it to take NF into account. -void MaybeReshapeCache(const size_t default_cols, MatPtrT& cache) { - if (default_cols == cache.Cols()) { - cache.ReshapePackedRowsToCols(2 * FloatsPerVector()); - } -} - -// Transposes a single row of the kv cache into the k-cache and v-cache. -void TransposeKVCacheRow(const KV_t* HWY_RESTRICT kv, KV_t* HWY_RESTRICT k, - KV_t* HWY_RESTRICT v, size_t qkv_dim) { - // This is inefficient, as the writes are scattered over cache lines, but it - // is a tiny fraction of the overall computation, and it is linear in the - // token length. - const size_t kFloatsPerTile = 2 * FloatsPerVector(); - const size_t kRoundedQkvDim = hwy::RoundUpTo(qkv_dim, kMaxBF16PerVector); - for (size_t i = 0; i < qkv_dim; i += 2) { - k[i * kFloatsPerTile] = kv[i]; - k[i * kFloatsPerTile + 1] = kv[i + 1]; - } - for (size_t i = qkv_dim; i < kRoundedQkvDim; i += 2) { - k[i * kFloatsPerTile] = hwy::ConvertScalarTo(0.0f); - k[i * kFloatsPerTile + 1] = hwy::ConvertScalarTo(0.0f); - } - for (size_t i = 0; i < qkv_dim; i += kFloatsPerTile) { - if (i + kFloatsPerTile <= qkv_dim) { - for (size_t j = 0; j < kFloatsPerTile; j++) { - v[i * kFloatsPerTile + j] = kv[i + j + qkv_dim]; - } - } else { - for (size_t j = 0; j < qkv_dim - i; j++) { - v[i * kFloatsPerTile + j] = kv[i + j + qkv_dim]; - } - for (size_t j = qkv_dim - i; j < kFloatsPerTile; j++) { - v[i * kFloatsPerTile + j] = hwy::ConvertScalarTo(0.0f); - } - } - } - for (size_t i = hwy::RoundUpTo(qkv_dim, kFloatsPerTile); i < kRoundedQkvDim; - i += kFloatsPerTile) { - for (size_t j = 0; j < kFloatsPerTile; j++) { - v[i * kFloatsPerTile + j] = hwy::ConvertScalarTo(0.0f); - } - } -} - -// Zeros out a part of k and v that corresponds to out-of-bounds cache -// positions. -void TransposeOOBKVCacheRow(KV_t* HWY_RESTRICT k, KV_t* HWY_RESTRICT v, - size_t qkv_dim) { - const size_t kFloatsPerTile = 2 * FloatsPerVector(); - const size_t kRoundedQkvDim = hwy::RoundUpTo(qkv_dim, kMaxBF16PerVector); - for (size_t i = 0; i < kRoundedQkvDim; i += 2) { - k[i * kFloatsPerTile] = hwy::ConvertScalarTo(0.0f); - k[i * kFloatsPerTile + 1] = hwy::ConvertScalarTo(0.0f); - } - for (size_t i = 0; i < kRoundedQkvDim; i += kFloatsPerTile) { - for (size_t j = 0; j < kFloatsPerTile; j++) { - v[i * kFloatsPerTile + j] = hwy::ConvertScalarTo(0.0f); - } - } -} - // Computes Q.K scores, which are "logits" (or scores) stored to att. // `k` is a strided view of the kv cache with dimensions [seq_len, qkv_dim]. static HWY_INLINE void QDotK(const size_t start_pos, const size_t last_pos, @@ -128,41 +57,45 @@ static HWY_INLINE void QDotK(const size_t start_pos, const size_t last_pos, const MatPtrT& k, float* HWY_RESTRICT att, ThreadingContext& ctx, const size_t worker) { GCPP_ZONE(ctx, worker, Zones::kGenAttentionQDotK); - const hn::ScalableTag dbf; - const size_t qkv_dim = k.Cols(); - HWY_ALIGN BF16 q_bf[kMaxQKVDim]; - - CompressPerThread tls; - const hn::ScalableTag df; - CompressTraits::Compress(df, q, qkv_dim, tls, MakeSpan(q_bf, qkv_dim), - 0); - - // --seq_len must be large enough to avoid wraparound. - HWY_DASSERT(last_pos < static_cast(div_seq_len.GetDivisor())); - for (size_t pos = start_pos; pos <= last_pos; ++pos) { - const float score = - Dot(dbf, MakeConstSpan(q_bf, qkv_dim), 0, k.Row(pos), qkv_dim); - att[pos] = score; + if (HWY_LIKELY(last_pos < static_cast(div_seq_len.GetDivisor()))) { + // Slightly faster: no wraparound. + for (size_t pos = start_pos; pos <= last_pos; ++pos) { + const float score = Dot(q, k.Row(pos), k.Cols()); + att[pos] = score; + } + } else { + for (size_t pos = start_pos; pos <= last_pos; ++pos) { + const size_t pos_modulo = div_seq_len.Remainder(pos); + const float score = Dot(q, k.Row(pos_modulo), k.Cols()); + att[pos_modulo] = score; + } } } void PositionalEncodingQK(float* qk, const size_t layer_idx, - const AttentionActivationsPtrs& activations, + const LayerWeightsPtrs& layer, + const AttentionActivations& activations, ThreadingContext& ctx, const size_t worker, const size_t pos, const float mul) { - const LayerConfig& layer_config = activations.config.layer_configs[layer_idx]; - const size_t qkv_dim = layer_config.qkv_dim; - const PostQKType& post_qk = layer_config.post_qk; + const size_t qkv_dim = layer.layer_config.qkv_dim; + const PostQKType& post_qk = layer.layer_config.post_qk; // qk is either q or k, so qkv_dim is the length we operate on. const float* inv_timescale = activations.inv_timescale.PackedScale1(); const bool is_global_layer = activations.config.IsGlobalLayer(layer_idx); - if (is_global_layer && activations.config.use_global_timescale) { + // Use global (full-attention) RoPE frequencies for VLM and Gemma 4. + if (is_global_layer && (IsVLM(activations.config.model) || + post_qk == PostQKType::PartialRope)) { inv_timescale = activations.inv_timescale_global.PackedScale1(); } // PostQKType::Rope if (post_qk == PostQKType::HalfRope) { Rope(qk, qkv_dim / 2, inv_timescale, pos, ctx, worker); if (mul != 1.0f) MulByConst(mul, qk, qkv_dim); + } else if (post_qk == PostQKType::PartialRope) { + // Gemma 4: partial rotary on first quarter of head dim for full attention. + const size_t rotary_dim = qkv_dim / 4; + Rope(qk, rotary_dim, inv_timescale, pos, ctx, worker); + if (mul != 1.0f) MulByConst(mul, qk, qkv_dim); } else { RopeAndMulBy(mul, qk, qkv_dim, inv_timescale, pos, ctx, worker); } @@ -177,52 +110,62 @@ static HWY_INLINE void WeightedSumV( const hwy::Divisor& div_seq_len, const float* HWY_RESTRICT att, const MatPtrT& v, float* HWY_RESTRICT att_out, ThreadingContext& ctx, const size_t worker) { - // --seq_len must be large enough to avoid wraparound. - HWY_DASSERT(last_pos < static_cast(div_seq_len.GetDivisor())); - // TODO: replace with MatMul(att, v) after it supports non-transposed B. - MulByConstTo(att[start_pos], v.Row(start_pos), att_out, v.Cols(), ctx, - worker); - for (size_t pos = start_pos + 1; pos <= last_pos; ++pos) { - MulByConstAndAdd(att[pos], v.Row(pos), att_out, v.Cols()); + if (HWY_LIKELY(last_pos < static_cast(div_seq_len.GetDivisor()))) { + // Slightly faster: no wraparound. Could be replaced with MatMul(att, v) if + // we supported non-transposed B. + // TODO: 2..4x unroll + MulByConstTo(att[start_pos], v.Row(start_pos), att_out, v.Cols(), ctx, + worker); + for (size_t pos = start_pos + 1; pos <= last_pos; ++pos) { + MulByConstAndAdd(att[pos], v.Row(pos), att_out, v.Cols()); + } + } else { + { + const size_t pos_mod = div_seq_len.Remainder(start_pos); + MulByConstTo(att[pos_mod], v.Row(pos_mod), att_out, v.Cols(), ctx, + worker); + } + for (size_t pos = start_pos + 1; pos <= last_pos; ++pos) { + const size_t pos_mod = div_seq_len.Remainder(pos); + MulByConstAndAdd(att[pos_mod], v.Row(pos_mod), att_out, v.Cols()); + } } } // Calculates the attention outputs for a single q, which may be updated // in place for RMSNorm. void SingleDotSoftmaxWeightedSum( - const size_t q_pos, const size_t kv_start_pos, const size_t kv_last_pos, + const size_t pos, const size_t start_pos, const size_t last_pos, float* HWY_RESTRICT q, const MatPtrT& k, const MatPtrT& v, - const MatPtr& query_norm_scale, const size_t layer_idx, - const AttentionActivationsPtrs& activations, float* HWY_RESTRICT att, - float* HWY_RESTRICT att_out, const SMOptions& sm_options, - ThreadingContext& ctx, const size_t worker) { + const size_t layer_idx, const LayerWeightsPtrs& layer, + const AttentionActivations& activations, float* HWY_RESTRICT att, + float* HWY_RESTRICT att_out, ThreadingContext& ctx, const size_t worker) { const float att_cap = activations.config.att_cap; const float query_scale = activations.query_scale; - // --seq_len must be large enough to avoid wraparound. - HWY_DASSERT(kv_last_pos < activations.SeqLen()); - const LayerConfig& layer_config = activations.config.layer_configs[layer_idx]; + const size_t seq_len = + static_cast(activations.div_seq_len.GetDivisor()); // Apply rope and scaling to Q. - if (query_norm_scale.HasPtr()) { - CallUpcasted(&query_norm_scale, [&](const auto* weights_t) { + if (layer.query_norm_scale.HasPtr()) { + CallUpcasted(&layer.query_norm_scale, [&](const auto* weights_t) { RMSNormInplace(weights_t->PackedScale1(), /*w_ofs=*/0, q, - layer_config.qkv_dim, ctx, worker); + layer.layer_config.qkv_dim, ctx, worker); }); } - PositionalEncodingQK(q, layer_idx, activations, ctx, worker, q_pos, + PositionalEncodingQK(q, layer_idx, layer, activations, ctx, worker, pos, query_scale); - QDotK(kv_start_pos, kv_last_pos, activations.div_seq_len, q, k, att, ctx, - worker); + QDotK(start_pos, last_pos, activations.div_seq_len, q, k, att, ctx, worker); // SoftMax with optional SoftCap yields "probabilities" in att. - const Logits logits(att, kv_last_pos + 1); + const size_t att_len = HWY_MIN(last_pos + 1, seq_len); + const Logits logits(att, att_len); MaybeLogitsSoftCap(att_cap, logits, ctx, worker); - Softmax(logits, ctx, worker, /*temperature=*/1.0f, sm_options); + Softmax(logits, ctx, worker, /*temperature=*/1.0f); - WeightedSumV(kv_start_pos, kv_last_pos, activations.div_seq_len, att, v, - att_out, ctx, worker); + WeightedSumV(start_pos, last_pos, activations.div_seq_len, att, v, att_out, + ctx, worker); } // The attention window usually starts at 0 unless `pos` is larger than @@ -233,13 +176,13 @@ size_t StartPos(size_t pos, const ModelConfig& config, size_t layer_idx) { } void DotSoftmaxWeightedSum(const size_t num_tokens, const size_t layer_idx, - const MatPtr& query_norm_scale, - AttentionActivationsPtrs& activations, - QBatch& qbatch, ThreadingContext& ctx) { + const LayerWeightsPtrs& layer, + AttentionActivations& activations, QBatch& qbatch, + ThreadingContext& ctx) { GCPP_ZONE(ctx, 0, Zones::kGenAttentionDotSoftmaxWeightedSumInclusive); const hwy::Divisor div_qbatch(qbatch.Size()); - const LayerConfig& layer_config = activations.config.layer_configs[layer_idx]; + const LayerConfig& layer_config = layer.layer_config; const size_t qkv_dim = layer_config.qkv_dim; // A "head group" in the context of GQA refers to a collection of query @@ -247,7 +190,8 @@ void DotSoftmaxWeightedSum(const size_t num_tokens, const size_t layer_idx, const size_t kHeadGroups = layer_config.heads / layer_config.kv_heads; const size_t cache_layer_size = layer_config.CacheLayerSize(); - const size_t seq_len = activations.SeqLen(); + const size_t seq_len = + static_cast(activations.div_seq_len.GetDivisor()); // All layers should have the same number of heads. HWY_DASSERT(activations.div_heads.GetDivisor() == layer_config.heads); @@ -258,12 +202,12 @@ void DotSoftmaxWeightedSum(const size_t num_tokens, const size_t layer_idx, GCPP_ZONE(ctx, worker, Zones::kGenAttentionDotSoftmaxWeightedSumPar); const size_t qi = div_qbatch.Remainder(tq_idx); - const size_t token_idx = div_qbatch.Divide(tq_idx); + const size_t batch_idx = div_qbatch.Divide(tq_idx); auto& kv_cache = qbatch.KV(qi).kv_cache; // Find the token position in the query and calculate // the range of cache positions to attend to. - const size_t pos = qbatch.Pos(qi) + token_idx; + const size_t pos = qbatch.Pos(qi) + batch_idx; const size_t start_pos = StartPos(pos, activations.config, layer_idx); size_t last_pos = pos; const size_t prefix_end = qbatch.PrefixEnd(qi); @@ -276,8 +220,6 @@ void DotSoftmaxWeightedSum(const size_t num_tokens, const size_t layer_idx, float* HWY_RESTRICT att = activations.att.Row(tq_idx) + head * seq_len; float* HWY_RESTRICT att_out = activations.att_out.Row(tq_idx) + head * qkv_dim; - SMOptions sm_options{.max_out = activations.softmax_max.Row(tq_idx) + head, - .d_out = activations.softmax_d.Row(tq_idx) + head}; // Make strided read-only views into the kv cache for // this query and head. @@ -288,10 +230,8 @@ void DotSoftmaxWeightedSum(const size_t num_tokens, const size_t layer_idx, MatPtrT v("v_view", Extents2D(seq_len, qkv_dim)); v.SetPtr(kv_cache.Row(0) + kv_head_offset + qkv_dim, kv_cache.Stride()); - constexpr size_t offset = 0; // placeholder, do not remove - SingleDotSoftmaxWeightedSum(pos + offset, start_pos, last_pos, q, k, v, - query_norm_scale, layer_idx, activations, att, - att_out, sm_options, ctx, worker); + SingleDotSoftmaxWeightedSum(pos, start_pos, last_pos, q, k, v, layer_idx, + layer, activations, att, att_out, ctx, worker); }; { @@ -312,7 +252,7 @@ void DotSoftmaxWeightedSum(const size_t num_tokens, const size_t layer_idx, // Fills activations.q and writes to KV cache. static HWY_INLINE void ComputeQKV(size_t num_tokens, const size_t layer_idx, const LayerWeightsPtrs& layer, - AttentionActivationsPtrs& activations, + AttentionActivations& activations, const QBatch& qbatch, const int flags, MatMulEnv& env) { GCPP_ZONE(env.ctx, hwy::Profiler::GlobalIdx(), @@ -337,76 +277,34 @@ static HWY_INLINE void ComputeQKV(size_t num_tokens, const size_t layer_idx, layer.qkv_einsum_w2.Rows())); for (size_t interleaved_idx = 0; interleaved_idx < num_interleaved; ++interleaved_idx) { - // Index into qbatch, within [0, qbatch.Size()] const size_t qi = div_qbatch.Remainder(interleaved_idx); - // Index along token sequence, within [0, num_tokens) - const size_t token_idx = div_qbatch.Divide(interleaved_idx); - const size_t cache_pos = qbatch.Pos(qi) + token_idx; - // --seq_len must be large enough to avoid wraparound. - HWY_DASSERT(cache_pos < activations.SeqLen()); - + const size_t batch_idx = div_qbatch.Divide(interleaved_idx); + const size_t cache_pos = + activations.div_seq_len.Remainder(qbatch.Pos(qi) + batch_idx); env.row_ptrs[0][interleaved_idx] = reinterpret_cast( qbatch.KV(qi).kv_cache.Row(cache_pos) + layer_idx * cache_layer_size); } kv_rows.AttachRowPtrs(env.row_ptrs[0].get()); CallMatMul(activations.pre_att_rms_out, layer.qkv_einsum_w2, /*add=*/nullptr, env, kv_rows); - for (size_t qi = 0; qi < qbatch.Size(); ++qi) { - MaybeReshapeCache(qbatch.KV(qi).cache->KOrVDefaultCols(), - qbatch.KV(qi).k_cache); - MaybeReshapeCache(qbatch.KV(qi).cache->KOrVDefaultCols(), - qbatch.KV(qi).v_cache); - } - const size_t kFloatsPerVector = FloatsPerVector(); - const size_t kRoundedTokens = - hwy::RoundUpTo(num_tokens, 2 * kFloatsPerVector); - const size_t kRoundedNumInterleaved = - kRoundedTokens * div_qbatch.GetDivisor(); // Apply positional encodings for K. // Note that 2D parallelism is not worth the fork/join overhead because the // tasks are very lightweight. ParallelFor( - Parallelism::kFlat, kv_heads * kRoundedNumInterleaved, env.ctx, + ParallelismStrategy::kFlat, kv_heads * num_interleaved, env.ctx, /*cluster_idx=*/0, Callers::kAttComputeQKV, [&](size_t task, size_t worker) HWY_ATTR { const size_t head = task % kv_heads; const size_t interleaved_idx = task / kv_heads; const size_t qi = div_qbatch.Remainder(interleaved_idx); - const size_t token_idx = div_qbatch.Divide(interleaved_idx); - const size_t cache_pos = qbatch.Pos(qi) + token_idx; - if (token_idx >= kRoundedTokens) { - return; - } - // The innermost dimension of v is 2NF values from qkv_dim because they - // will be loaded into a BF16 vector to be scaled and added to the - // cached attention output in 2 NF-sized registers. - auto& k_cache = qbatch.KV(qi).k_cache; - KV_t* HWY_RESTRICT k = - k_cache.Row(cache_pos / (2 * kFloatsPerVector)) + - qbatch.KV(qi).cache->KOffset(layer_idx, head, kFloatsPerVector, - cache_pos); - auto& v_cache = qbatch.KV(qi).v_cache; - KV_t* HWY_RESTRICT v = - v_cache.Row(cache_pos / (2 * kFloatsPerVector)) + - qbatch.KV(qi).cache->VOffset(layer_idx, head, kFloatsPerVector, - cache_pos); - if (token_idx >= num_tokens) { - // Create a zero-filled K/V pair for padding for out-of-sequence - // tokens. - TransposeOOBKVCacheRow(k, v, qkv_dim); - return; - } - // --seq_len must be large enough to avoid wraparound. - HWY_DASSERT(cache_pos < activations.SeqLen()); + const size_t batch_idx = div_qbatch.Divide(interleaved_idx); + const size_t pos = qbatch.Pos(qi) + batch_idx; + const size_t cache_pos = activations.div_seq_len.Remainder(pos); auto& kv_cache = qbatch.KV(qi).kv_cache; KV_t* HWY_RESTRICT kv = kv_cache.Row(cache_pos) + layer_idx * cache_layer_size + head * qkv_dim * 2; - // Note that k_cache and v_cache are different shapes. - // The innermost dimension of k is 2 values from qkv_dim because they - // are going to be used in a BF16 dot product involving pairs of - // values over NF k positions. HWY_ALIGN float kv_f32[2 * kMaxQKVDim]; const hn::ScalableTag df; @@ -421,23 +319,43 @@ static HWY_INLINE void ComputeQKV(size_t num_tokens, const size_t layer_idx, }); } - constexpr size_t offset = 0; // placeholder, do not remove - PositionalEncodingQK(kv_f32, layer_idx, activations, env.ctx, worker, - cache_pos + offset, - /*mul=*/1.0f); + PositionalEncodingQK(kv_f32, layer_idx, layer, activations, env.ctx, + worker, pos, /*mul=*/1.0f); + + // Gemma 4: apply per-layer KV scaling if present. + if (layer.attn_kv_scale.HasPtr()) { + CallUpcasted(&layer.attn_kv_scale, [&](const auto* weights_t) { + MulByConst(weights_t->PackedScale1()[0], kv_f32, 2 * qkv_dim); + }); + } CompressPerThread tls; Compress(kv_f32, 2 * qkv_dim, tls, MakeSpan(kv, 2 * qkv_dim), 0); - // This is inefficient, as multiple threads are writing the same K - // cache line, but the input is generated by a matmul, so it is - // difficult to change, and it probably isn't significant. - TransposeKVCacheRow(kv, k, v, qkv_dim); }); } +// Sums encoded (`att_out`) over num_heads (`layer_config.heads`) and +// head_dim (`qkv_dim`) into output (`layer_out`). +static HWY_INLINE void SumHeads(const LayerWeightsPtrs& layer, + AttentionActivations& activations, + MatMulEnv& env) { + GCPP_ZONE(env.ctx, hwy::Profiler::GlobalIdx(), Zones::kGenAttentionSumHeads); + const LayerConfig& layer_config = layer.layer_config; + (void)layer_config; // For HWY_DASSERT + // att_weights and att_out are concatenated heads, each of length + // layer_config.qkv_dim. Thus the [num_interleaved, + // layer_config.model_dim] matmul output is the sum over heads. Compare + // gemma/modules.py: attn_output = self.attn_vec_einsum('BTNH,NHD->BTD', + // encoded) + HWY_DASSERT(layer_config.model_dim != 0 && layer_config.heads != 0 && + layer_config.qkv_dim != 0); + CallMatMul(activations.att_out, layer.att_weights, /*add=*/nullptr, env, + activations.att_sums); +} + void GemmaAttention(size_t num_tokens, const size_t layer_idx, const LayerWeightsPtrs& layer, - AttentionActivationsPtrs& activations, QBatch& qbatch, - MatMulEnv& env, AttentionImpl attention_impl, int flags) { + AttentionActivations& activations, QBatch& qbatch, + MatMulEnv& env, int flags) { GCPP_ZONE(env.ctx, hwy::Profiler::GlobalIdx(), Zones::kGenAttention); const LayerConfig& layer_config = layer.layer_config; @@ -447,16 +365,14 @@ void GemmaAttention(size_t num_tokens, const size_t layer_idx, (void)layer_config; // only used in HWY_DASSERT ComputeQKV(num_tokens, layer_idx, layer, activations, qbatch, flags, env); - if (attention_impl == AttentionImpl::kOld) { - DotSoftmaxWeightedSum(num_tokens, layer_idx, layer.query_norm_scale, - activations, qbatch, env.ctx); + if (flags & kAttentionUseOld) { + DotSoftmaxWeightedSum(num_tokens, layer_idx, layer, activations, qbatch, + env.ctx); } else { // * 2 does not help on Turin. FlashAttention(num_tokens, - /*target_parallelism=*/env.ctx.pools.MaxWorkers() * - AttentionActivations::kThreadReplicationFactor, - layer_idx, layer.query_norm_scale, activations, qbatch, - env.ctx, attention_impl); + /*target_parallelism=*/env.ctx.pools.MaxWorkers() * 1, + layer_idx, layer, activations, qbatch, env.ctx); } SumHeads(layer, activations, env); } diff --git a/gemma/configs.cc b/gemma/configs.cc index 0a585da5..4f717579 100644 --- a/gemma/configs.cc +++ b/gemma/configs.cc @@ -19,12 +19,11 @@ #include #include -#include #include #include "compression/types.h" // Type -#include "io/fields.h" // IFields -#include "io/io.h" // Path +#include "io/fields.h" // IFields +#include "io/io.h" // Path #include "hwy/base.h" namespace gcpp { @@ -239,7 +238,6 @@ static ModelConfig ConfigGemma3_1B() { config.display_name = "Gemma3_1B"; config.model = Model::GEMMA3_1B; config.wrapping = PromptWrapping::GEMMA_VLM; - config.use_global_timescale = true; config.model_dim = 1152; config.vocab_size = kGemmaV3VocabSize; // new vocab size / tokenizer config.max_seq_len = 32 * 1024; @@ -290,7 +288,6 @@ static ModelConfig ConfigGemma3_4B() { config.display_name = "Gemma3_4B"; config.model = Model::GEMMA3_4B; config.wrapping = PromptWrapping::GEMMA_VLM; - config.use_global_timescale = true; AddVitConfig(config, /*image_size=*/896); config.vocab_size = kGemmaV3VocabSize; config.vit_config.pool_dim = 4; @@ -340,7 +337,6 @@ static ModelConfig ConfigGemma3_12B() { config.display_name = "Gemma3_12B"; config.model = Model::GEMMA3_12B; config.wrapping = PromptWrapping::GEMMA_VLM; - config.use_global_timescale = true; AddVitConfig(config, /*image_size=*/896); config.vocab_size = kGemmaV3VocabSize; config.vit_config.pool_dim = 4; @@ -390,7 +386,6 @@ static ModelConfig ConfigGemma3_27B() { config.display_name = "Gemma3_27B"; config.model = Model::GEMMA3_27B; config.wrapping = PromptWrapping::GEMMA_VLM; - config.use_global_timescale = true; AddVitConfig(config, /*image_size=*/896); config.vocab_size = kGemmaV3VocabSize; config.vit_config.pool_dim = 4; @@ -461,11 +456,202 @@ static ModelConfig ConfigFromModel(Model model) { return ConfigGemma3_27B(); case Model::GEMMA3_270M: return ConfigGemma3_270M(); + case Model::GEMMA4_E2B: + return ConfigGemma4_E2B(); + case Model::GEMMA4_E4B: + return ConfigGemma4_E4B(); + case Model::GEMMA4_26B_A4B: + return ConfigGemma4_26B_A4B(); + case Model::GEMMA4_31B: + return ConfigGemma4_31B(); default: HWY_ABORT("Model type %d unknown.", static_cast(model)); } } +// Gemma 4 Configurations +static constexpr size_t kGemma4VocabSize = 262144; + +static LayerConfig LayerConfigGemma4_E2B(size_t model_dim) { + LayerConfig config; + config.model_dim = model_dim; + config.ff_hidden_dim = 6144; + config.heads = 8; + config.kv_heads = 1; + config.qkv_dim = 256; + config.optimized_gating = true; + config.post_norm = PostNormType::Scale; + config.use_qk_norm = true; + config.hidden_size_per_layer_input = 256; + config.num_kv_shared_layers = 20; + config.use_double_wide_mlp = true; + return config; +} + +static ModelConfig ConfigGemma4_E2B() { + ModelConfig config = ConfigBaseGemmaV3(); + config.display_name = "Gemma4_E2B"; + config.model = Model::GEMMA4_E2B; + config.wrapping = PromptWrapping::GEMMA_VLM; + config.model_dim = 1536; + config.vocab_size = kGemma4VocabSize; + config.max_seq_len = 128 * 1024; + LayerConfig lc = LayerConfigGemma4_E2B(config.model_dim); + config.num_layers = 35; + config.layer_configs = {config.num_layers, lc}; + // Configure full-attention layers with larger head dims and fewer KV heads. + for (size_t i = 0; i < config.layer_configs.size(); ++i) { + if (config.IsGlobalLayer(i)) { + config.layer_configs[i].post_qk = PostQKType::PartialRope; + config.layer_configs[i].qkv_dim = 512; + config.layer_configs[i].kv_heads = config.num_global_kv_heads; + } + } + config.query_scale = QueryScaleType::SqrtKeySize; + config.attention_window_sizes = RepeatedAttentionWindowSizes<35, 5>( + {512, 512, 512, 512, config.max_seq_len}); + config.final_logit_softcapping = 30.0f; + config.rope_theta_sliding = 10000.0f; + config.rope_theta_full = 1000000.0f; + config.partial_rotary_factor = 0.25f; + return config; +} + +static LayerConfig LayerConfigGemma4_E4B(size_t model_dim) { + LayerConfig config; + config.model_dim = model_dim; + config.ff_hidden_dim = 10240; + config.heads = 8; + config.kv_heads = 2; + config.qkv_dim = 256; + config.optimized_gating = true; + config.post_norm = PostNormType::Scale; + config.use_qk_norm = true; + config.hidden_size_per_layer_input = 256; + config.num_kv_shared_layers = 18; + return config; +} + +static ModelConfig ConfigGemma4_E4B() { + ModelConfig config = ConfigBaseGemmaV3(); + config.display_name = "Gemma4_E4B"; + config.model = Model::GEMMA4_E4B; + config.wrapping = PromptWrapping::GEMMA_VLM; + config.model_dim = 2560; + config.vocab_size = kGemma4VocabSize; + config.max_seq_len = 128 * 1024; + LayerConfig lc = LayerConfigGemma4_E4B(config.model_dim); + config.num_layers = 42; + config.layer_configs = {config.num_layers, lc}; + // Configure full-attention layers with larger head dims and fewer KV heads. + for (size_t i = 0; i < config.layer_configs.size(); ++i) { + if (config.IsGlobalLayer(i)) { + config.layer_configs[i].post_qk = PostQKType::PartialRope; + config.layer_configs[i].qkv_dim = 512; + config.layer_configs[i].kv_heads = config.num_global_kv_heads; + } + } + config.query_scale = QueryScaleType::SqrtKeySize; + config.attention_window_sizes = RepeatedAttentionWindowSizes<42, 6>( + {512, 512, 512, 512, 512, config.max_seq_len}); + config.final_logit_softcapping = 30.0f; + config.rope_theta_sliding = 10000.0f; + config.rope_theta_full = 1000000.0f; + config.partial_rotary_factor = 0.25f; + return config; +} + +static LayerConfig LayerConfigGemma4_26B_A4B(size_t model_dim) { + LayerConfig config; + config.model_dim = model_dim; + config.ff_hidden_dim = 2112; + config.heads = 16; + config.kv_heads = 8; + config.qkv_dim = 256; + config.optimized_gating = true; + config.post_norm = PostNormType::Scale; + config.use_qk_norm = true; + config.num_experts = 128; + config.top_k_experts = 8; + config.moe_intermediate_size = 704; + config.num_global_kv_heads = 2; + config.global_head_dim = 512; + return config; +} + +static ModelConfig ConfigGemma4_26B_A4B() { + ModelConfig config = ConfigBaseGemmaV3(); + config.display_name = "Gemma4_26B_A4B"; + config.model = Model::GEMMA4_26B_A4B; + config.wrapping = PromptWrapping::GEMMA_VLM; + config.model_dim = 2816; + config.vocab_size = kGemma4VocabSize; + config.max_seq_len = 256 * 1024; + LayerConfig lc = LayerConfigGemma4_26B_A4B(config.model_dim); + config.num_layers = 30; + config.layer_configs = {config.num_layers, lc}; + // Configure full-attention layers with larger head dims and fewer KV heads. + for (size_t i = 0; i < config.layer_configs.size(); ++i) { + if (config.IsGlobalLayer(i)) { + config.layer_configs[i].post_qk = PostQKType::PartialRope; + config.layer_configs[i].qkv_dim = 512; + config.layer_configs[i].kv_heads = config.num_global_kv_heads; + } + } + config.query_scale = QueryScaleType::SqrtKeySize; + config.attention_window_sizes = RepeatedAttentionWindowSizes<30, 6>( + {1024, 1024, 1024, 1024, 1024, config.max_seq_len}); + config.final_logit_softcapping = 30.0f; + config.rope_theta_sliding = 10000.0f; + config.rope_theta_full = 1000000.0f; + config.partial_rotary_factor = 0.25f; + return config; +} + +static LayerConfig LayerConfigGemma4_31B(size_t model_dim) { + LayerConfig config; + config.model_dim = model_dim; + config.ff_hidden_dim = 21504; + config.heads = 32; + config.kv_heads = 16; + config.qkv_dim = 256; + config.optimized_gating = true; + config.post_norm = PostNormType::Scale; + config.use_qk_norm = true; + config.num_global_kv_heads = 4; + config.global_head_dim = 512; + return config; +} + +static ModelConfig ConfigGemma4_31B() { + ModelConfig config = ConfigBaseGemmaV3(); + config.display_name = "Gemma4_31B"; + config.model = Model::GEMMA4_31B; + config.wrapping = PromptWrapping::GEMMA_VLM; + config.model_dim = 5376; + config.vocab_size = kGemma4VocabSize; + config.max_seq_len = 256 * 1024; + LayerConfig lc = LayerConfigGemma4_31B(config.model_dim); + config.num_layers = 60; + config.layer_configs = {config.num_layers, lc}; + // Configure full-attention layers with larger head dims and fewer KV heads. + for (size_t i = 0; i < config.layer_configs.size(); ++i) { + if (config.IsGlobalLayer(i)) { + config.layer_configs[i].post_qk = PostQKType::PartialRope; + config.layer_configs[i].qkv_dim = 512; + config.layer_configs[i].kv_heads = config.num_global_kv_heads; + } + } + config.query_scale = QueryScaleType::SqrtKeySize; + config.attention_window_sizes = RepeatedAttentionWindowSizes<60, 6>( + {1024, 1024, 1024, 1024, 1024, config.max_seq_len}); + config.final_logit_softcapping = 30.0f; + config.rope_theta_sliding = 10000.0f; + config.rope_theta_full = 1000000.0f; + config.partial_rotary_factor = 0.25f; + return config; +} + const char* ModelPrefix(Model model) { switch (model) { case Model::UNKNOWN: @@ -494,25 +680,33 @@ const char* ModelPrefix(Model model) { return "gemma3-27b"; case Model::GEMMA3_270M: return "gemma3-270m"; + case Model::GEMMA4_E2B: + return "gemma4-e2b"; + case Model::GEMMA4_E4B: + return "gemma4-e4b"; + case Model::GEMMA4_26B_A4B: + return "gemma4-26b-a4b"; + case Model::GEMMA4_31B: + return "gemma4-31b"; default: HWY_ABORT("Model type %d unknown.", static_cast(model)); } } PromptWrapping ChooseWrapping(const Model model, Tristate wrapping) { - const PromptWrapping config_wrapping = ConfigFromModel(model).wrapping; - - // For models with a fixed wrapping mode, ignore user override. - if (config_wrapping == PromptWrapping::PALIGEMMA || - config_wrapping == PromptWrapping::GEMMA_VLM) { + if (IsPaliGemma(model)) { if (wrapping != Tristate::kDefault) { - HWY_WARN("Ignoring unnecessary --wrapping for model %s.", - ModelPrefix(model)); + HWY_WARN("Ignoring unnecessary --wrapping for PaliGemma models."); } - return config_wrapping; + return PromptWrapping::PALIGEMMA; } - - // For other models, default to IT unless --wrapping=0 is passed. + if (IsVLM(model)) { + if (wrapping != Tristate::kDefault) { + HWY_WARN("Ignoring unnecessary --wrapping for VLM models."); + } + return PromptWrapping::GEMMA_VLM; + } + // Default to IT unless --wrapping=0. return wrapping == Tristate::kFalse ? PromptWrapping::GEMMA_PT : PromptWrapping::GEMMA_IT; } @@ -679,15 +873,17 @@ Model DeduceModel(const Path& blob_path, size_t layers, int layer_types) { return Model::GEMMA3_270M; case 26: - if (layer_types & (kDeducedViT | kDeducedKqNorm)) { - return Model::GEMMA3_1B; - } + if (layer_types & kDeducedViT) return Model::GEMMA3_1B; return Model::GEMMA2_2B; case 27: return (layer_types & kDeduced448) ? Model::PALIGEMMA2_3B_448 : Model::PALIGEMMA2_3B_224; + case 30: + return Model::GEMMA4_26B_A4B; case 34: return Model::GEMMA3_4B; + case 35: + return Model::GEMMA4_E2B; case 42: if (layer_types & kDeducedViT) { return (layer_types & kDeduced448) ? Model::PALIGEMMA2_10B_448 @@ -698,6 +894,8 @@ Model DeduceModel(const Path& blob_path, size_t layers, int layer_types) { return Model::GEMMA2_27B; case 48: return Model::GEMMA3_12B; + case 60: + return Model::GEMMA4_31B; case 62: return Model::GEMMA3_27B; @@ -713,28 +911,4 @@ Model DeduceModel(const Path& blob_path, size_t layers, int layer_types) { } } -constexpr std::pair kAttentionImplNameToEnum[] = { - {"old", AttentionImpl::kOld}, - {"flash", AttentionImpl::kFlash}, - {"flash_transposed_qs", AttentionImpl::kFlashTransposedQs}, - {"flash_transposed_qs_bf16", AttentionImpl::kFlashTransposedQsBF16}, - {"flash_transposed_qs_int16", AttentionImpl::kFlashTransposedQsInt16}, -}; - -std::string GetAttentionImplName(AttentionImpl impl) { - for (const auto& [name, attention_impl] : kAttentionImplNameToEnum) { - if (attention_impl == impl) return std::string(name); - } - return "unknown"; -} - -AttentionImpl GetAttentionImpl(const std::string& impl_name) { - for (const auto& [name, attention_impl] : kAttentionImplNameToEnum) { - if (name == impl_name) return attention_impl; - } - HWY_WARN("Unknown attention implementation: %s. Using kOld.\n", - impl_name.c_str()); - return AttentionImpl::kOld; -} - } // namespace gcpp diff --git a/gemma/configs.h b/gemma/configs.h index c14455ea..e3c65fb5 100644 --- a/gemma/configs.h +++ b/gemma/configs.h @@ -29,12 +29,9 @@ #include "io/fields.h" // IFieldsVisitor #include "io/io.h" // Path #include "util/basics.h" -#include "hwy/detect_compiler_arch.h" namespace gcpp { -constexpr size_t kMaxBF16PerVector = HWY_ARCH_MAX_BYTES / sizeof(BF16); - HWY_INLINE_VAR constexpr int kAttentionUseOld = 2; HWY_INLINE_VAR constexpr size_t kMaxQKVDim = 1024; @@ -83,55 +80,6 @@ static inline bool EnumValid(LayerAttentionType type) { return type == LayerAttentionType::kGemma || type == LayerAttentionType::kVit; } -// Values stated explicitly to allow for semantic reordering -enum class KVEncoding { - kUnspecified = 0, - kF32 = 1, - kBF16 = 2, - kF32TwoTranspositions = 3, - kBF16TwoTranspositions = 4, - kInt8 = 5, - kInt8TwoTranspositions = 6, -}; - -enum class AttentionImpl { - kOld, // Previous Attention implementation - kFlash, // Flash Attention (default) - kFlashTransposedQs, - kFlashTransposedQsBF16, - kFlashTransposedQsInt16, - kSentinel, -}; - -std::string GetAttentionImplName(AttentionImpl impl); -AttentionImpl GetAttentionImpl(const std::string& impl); - -/* - * Returns a bitmask of flags to pass to attention functions based on the - * attention implementation selected. - * - * If `hwy_native_dot_bf16` is true, the function will use the old attention - * implementation, ignoring `impl`. - * - * `hwy_native_dot_bf16` needs to be passed in, because the HWY_NATIVE_DOT_BF16 - * macro is not available outside of highway instrumented translation units and - * cannot be made accessible from .h files. - */ -static inline int AttentionImplToFlags(AttentionImpl impl, - int hwy_native_dot_bf16) { - if (hwy_native_dot_bf16) return kAttentionUseOld; - - switch (impl) { - case AttentionImpl::kOld: - return kAttentionUseOld; - case AttentionImpl::kFlash: - case AttentionImpl::kFlashTransposedQs: - case AttentionImpl::kFlashTransposedQsBF16: - default: - return 0; - } -} - // Post attention and ffw normalization type. enum class PostNormType { None, @@ -148,6 +96,7 @@ static inline bool EnumValid(PostNormType type) { enum class PostQKType { Rope, HalfRope, + PartialRope, // Gemma 4: partial rotary for full attention layers kSentinel // must be last }; @@ -229,7 +178,11 @@ enum class Model { GEMMA3_12B, GEMMA3_27B, GEMMA3_270M, - CUSTOM, + // Gemma 4 models + GEMMA4_E2B, + GEMMA4_E4B, + GEMMA4_26B_A4B, + GEMMA4_31B, kSentinel, }; @@ -237,6 +190,20 @@ enum class Model { // in Specifier and thus does not change. const char* ModelPrefix(Model model); +// Gemma3 is multimodal and has a different prompt wrapping than PaliGemma. +// This is used for deducing the PromptWrapping for pre-2025 BlobStore. +static inline bool IsVLM(Model model) { + return model == Model::GEMMA3_4B || model == Model::GEMMA3_1B || + model == Model::GEMMA3_12B || model == Model::GEMMA3_27B || + IsGemma4(model); +} + +// Gemma 4 is multimodal and has a different prompt wrapping. +static inline bool IsGemma4(Model model) { + return model == Model::GEMMA4_E2B || model == Model::GEMMA4_E4B || + model == Model::GEMMA4_26B_A4B || model == Model::GEMMA4_31B; +} + static inline bool IsPaliGemma(Model model) { if (model == Model::PALIGEMMA2_3B_224 || model == Model::PALIGEMMA2_3B_448 || model == Model::PALIGEMMA2_10B_224 || @@ -252,13 +219,13 @@ static inline bool IsObsolete(Model model) { return false; } -// Visits every valid model enum, skipping `UNKNOWN`, `kSentinel` and `CUSTOM`. +// Visits every valid model enum, skipping `UNKNOWN` and `kSentinel`. template void ForEachModel(const Func& func) { for (size_t i = static_cast(Model::GEMMA2_9B); i < static_cast(Model::kSentinel); ++i) { const Model model = static_cast(i); - if (!IsObsolete(model) && model != Model::CUSTOM) func(model); + if (!IsObsolete(model)) func(model); } } @@ -308,6 +275,14 @@ struct LayerConfig : public IFields { visitor(activation); visitor(post_qk); visitor(use_qk_norm); + visitor(num_experts); + visitor(top_k_experts); + visitor(moe_intermediate_size); + visitor(num_global_kv_heads); + visitor(global_head_dim); + visitor(hidden_size_per_layer_input); + visitor(num_kv_shared_layers); + visitor(use_double_wide_mlp); internal.VisitFields(visitor); // Append new fields here, then update `python/configs.cc`. } @@ -326,12 +301,21 @@ struct LayerConfig : public IFields { uint32_t kv_heads = 0; uint32_t qkv_dim = 0; // length of Q, K, V vectors (contiguous). bool ff_biases = false; - bool optimized_gating = true; // for Gemma3 + bool optimized_gating = true; // for Gemma3 PostNormType post_norm = PostNormType::None; LayerAttentionType type = LayerAttentionType::kGemma; ActivationType activation = ActivationType::Gelu; PostQKType post_qk = PostQKType::Rope; bool use_qk_norm = false; + // Gemma 4 fields + uint32_t num_experts = 0; // MoE: total experts (0 = dense) + uint32_t top_k_experts = 0; // MoE: active experts per token + uint32_t moe_intermediate_size = 0; // MoE: expert FFN dim + uint32_t num_global_kv_heads = 0; // Global attention KV heads + uint32_t global_head_dim = 0; // Global attention head dim + uint32_t hidden_size_per_layer_input = 0; // PLE per-layer input dim + uint32_t num_kv_shared_layers = 0; // KV cache sharing + bool use_double_wide_mlp = false; // E2B double-wide MLP InternalLayerConfig internal; }; @@ -411,6 +395,10 @@ struct ModelConfig : public IFields { visitor(att_cap); visitor(final_cap); + visitor(final_logit_softcapping); + visitor(rope_theta_sliding); + visitor(rope_theta_full); + visitor(partial_rotary_factor); visitor(absolute_pe); bool unused_use_local_attention = false; // formerly used for Griffin @@ -429,8 +417,6 @@ struct ModelConfig : public IFields { internal.VisitFields(visitor); - visitor(use_global_timescale); - // Append new fields here, then update `python/configs.cc`. } @@ -449,18 +435,6 @@ struct ModelConfig : public IFields { // The third ctor also expects a string returned by this. std::string Specifier() const; - // Overwrites `max_seq_len` with `new_max_seq_len` and updates all global - // layers' attention window sizes to `new_max_seq_len`. This function must be - // called before instantiating the KVCache object. - void SetMaxSeqLen(size_t new_max_seq_len) { - for (size_t i = 0; i < attention_window_sizes.size(); ++i) { - if (attention_window_sizes[i] == max_seq_len) { - attention_window_sizes[i] = new_max_seq_len; - } - } - max_seq_len = new_max_seq_len; - } - void AddLayerConfig(const LayerConfig& layer_config) { layer_configs.push_back(layer_config); HWY_ASSERT(layer_configs.size() <= num_layers); @@ -522,6 +496,11 @@ struct ModelConfig : public IFields { float att_cap = 0.0f; float final_cap = 0.0f; + // Gemma 4 fields + float final_logit_softcapping = 0.0f; // Logit soft capping (30.0 for Gemma 4) + float rope_theta_sliding = 10000.0f; // RoPE theta for sliding attention + float rope_theta_full = 1000000.0f; // RoPE theta for full attention + float partial_rotary_factor = 1.0f; // Partial rotary factor for full attention bool absolute_pe = false; QueryScaleType query_scale = QueryScaleType::SqrtKeySize; @@ -541,7 +520,6 @@ struct ModelConfig : public IFields { std::vector scale_base_names; InternalModelConfig internal; - bool use_global_timescale = false; // for Gemma 3 }; // Returns the sub-config for the ViT model of the PaliGemma model. @@ -549,8 +527,7 @@ ModelConfig GetVitConfig(const ModelConfig& config); enum DeducedLayerTypes { kDeducedViT = 2, - kDeduced448 = 4, // For ViT, 448x448 resolution instead of 224x224. - kDeducedKqNorm = 8, + kDeduced448 = 4, // For ViT, 448x448 resolution instead of 224x224. }; // layer_types is one or more of `DeducedLayerTypes`. diff --git a/gemma/gemma-inl.h b/gemma/gemma-inl.h index 0cd364a4..a56202bb 100644 --- a/gemma/gemma-inl.h +++ b/gemma/gemma-inl.h @@ -20,7 +20,6 @@ #include "gemma/activations.h" #include "gemma/configs.h" -#include "gemma/tensor_stats.h" #include "gemma/weights.h" #include "ops/matmul.h" #include "util/mat.h" @@ -71,11 +70,10 @@ template void ActivationBatched( ActivationType activation, Mat& c1, ThreadingContext& ctx, size_t cluster_idx = 0, - Parallelism parallelism = Parallelism::kFlat) { + ParallelismStrategy parallelism = ParallelismStrategy::kFlat) { using T = typename Mat::T; ParallelFor(parallelism, c1.Rows(), ctx, cluster_idx, Callers::kActivationBatched, [&](uint64_t task, size_t worker) { - // Cast to correct type so type deduction works. Activation(activation, c1.Row(task), static_cast(nullptr), c1.Cols(), ctx, worker); @@ -84,20 +82,17 @@ void ActivationBatched( #if GEMMA_FUSED_FFN -// Called during `TwoMatMul`. +// Called during TwoMatMul. static inline void Activation(ActivationType activation, const RowPtrsBF C1, const IndexRange range_r, const IndexRange range_c, const StridedViewBF C2, ThreadingContext& ctx, const size_t worker) { GCPP_ZONE(ctx, worker, Zones::kGenActivationFused); - const size_t cols = range_c.Num(); HWY_DASSERT(C2.Cols() == cols); - namespace hn = hwy::HWY_NAMESPACE; using DF = hn::ScalableTag; using VF = hn::Vec; - // ActivationType::Gelu // Gated: Gelu(c1) * c2. for (size_t ir = 0; ir < range_r.Num(); ++ir) { Decompress1AndCompressInplace( @@ -116,7 +111,7 @@ template HWY_NOINLINE void ActivationBatched( ActivationType activation, Mat1& c1, const Mat2* c2, ThreadingContext& ctx, size_t cluster_idx = 0, - Parallelism parallelism = Parallelism::kFlat) { + ParallelismStrategy parallelism = ParallelismStrategy::kFlat) { HWY_DASSERT(c1.SameShape(*c2)); if (c2 && c2->HasPtr()) { ParallelFor(parallelism, c1.Rows(), ctx, cluster_idx, @@ -152,6 +147,135 @@ void PostNorm(PostNormType post_norm, const MatPtr& weights, } } +// Gemma 4 MoE feed-forward with top-k routing. +static inline void FFWMoE(const LayerWeightsPtrs& layer, + Activations& activations, MatMulEnv& env) { + GCPP_ZONE(env.ctx, hwy::Profiler::GlobalIdx(), Zones::kGenFFW); + const LayerConfig& layer_config = layer.layer_config; + const size_t num_experts = layer_config.num_experts; + const size_t top_k = layer_config.top_k_experts; + const size_t model_dim = layer_config.model_dim; + const size_t batch_size = activations.pre_ffw_rms_out.Rows(); + + // Compute router logits: [batch, num_experts]. + CallMatMul(activations.pre_ffw_rms_out, layer.ffn_gate_in_w, + /*add=*/nullptr, env, activations.C1); + + // Find top-k experts per token and compute softmax routing weights. + // C2 stores [expert_idx_0, weight_0, expert_idx_1, weight_1, ...]. + ParallelFor(ParallelismStrategy::kFlat, batch_size, env.ctx, + /*cluster_idx=*/0, Callers::kSampleAndStream, + [&](size_t bi, size_t /*worker*/) { + float* router_logits = activations.C1.Row(bi); + float* routing = activations.C2.Row(bi); + // Selection sort for top-k. + for (size_t k = 0; k < top_k; ++k) { + size_t best_idx = 0; + float best_val = -1e30f; + for (size_t e = 0; e < num_experts; ++e) { + bool done = false; + for (size_t j = 0; j < k; ++j) { + if (static_cast(routing[j * 2]) == e) { done = true; break; } + } + if (!done && router_logits[e] > best_val) { + best_val = router_logits[e]; + best_idx = e; + } + } + routing[k * 2] = static_cast(best_idx); + routing[k * 2 + 1] = best_val; + } + // Softmax over top-k logits. + float max_val = routing[1]; + for (size_t k = 1; k < top_k; ++k) { + if (routing[k * 2 + 1] > max_val) max_val = routing[k * 2 + 1]; + } + float sum_exp = 0.0f; + for (size_t k = 0; k < top_k; ++k) { + routing[k * 2 + 1] = expf(routing[k * 2 + 1] - max_val); + sum_exp += routing[k * 2 + 1]; + } + for (size_t k = 0; k < top_k; ++k) routing[k * 2 + 1] /= sum_exp; + }); + + // Save routing data before the expert loop overwrites C2. + // Each token stores [expert_idx_0, weight_0, ..., expert_idx_k, weight_k]. + const size_t routing_cols = top_k * 2; + std::vector routing_data(batch_size * routing_cols); + for (size_t bi = 0; bi < batch_size; ++bi) { + const float* src = activations.C2.Row(bi); + float* dst = routing_data.data() + bi * routing_cols; + for (size_t c = 0; c < routing_cols; ++c) dst[c] = src[c]; + } + + // Zero out the output buffer. + memset(activations.ffw_out.PackedData(), 0, + batch_size * model_dim * sizeof(float)); + + // Process each expert: gather tokens, compute, scatter weighted results. + for (size_t expert_idx = 0; expert_idx < num_experts; ++expert_idx) { + // Count tokens routed to this expert. + size_t expert_token_count = 0; + for (size_t bi = 0; bi < batch_size; ++bi) { + const float* routing = routing_data.data() + bi * routing_cols; + for (size_t k = 0; k < top_k; ++k) { + if (static_cast(routing[k * 2]) == expert_idx) { + ++expert_token_count; + break; + } + } + } + if (expert_token_count == 0) continue; + + // Gather tokens for this expert, scaled by routing weight. + size_t write_idx = 0; + for (size_t bi = 0; bi < batch_size; ++bi) { + const float* routing = routing_data.data() + bi * routing_cols; + for (size_t k = 0; k < top_k; ++k) { + if (static_cast(routing[k * 2]) == expert_idx) { + const float weight = routing[k * 2 + 1]; + const float* input = activations.pre_ffw_rms_out.Row(bi); + float* dest = activations.C1.Row(write_idx); + for (size_t d = 0; d < model_dim; ++d) dest[d] = input[d] * weight; + ++write_idx; + break; + } + } + } + + // Expert FFN: gate * up, then down. + CallMatMul(activations.C1, layer.ffn_gate_w, + /*add=*/nullptr, env, activations.C1); + CallMatMul(activations.pre_ffw_rms_out, layer.ffn_up_w, + /*add=*/nullptr, env, activations.C2); + ActivationBatched(layer_config.activation, activations.C1, + &activations.C2, env.ctx); + CallMatMul(activations.C1, layer.ffn_down_w, + /*add=*/nullptr, env, activations.C1); + + // Scatter results back to output buffer. + write_idx = 0; + for (size_t bi = 0; bi < batch_size; ++bi) { + const float* routing = routing_data.data() + bi * routing_cols; + for (size_t k = 0; k < top_k; ++k) { + if (static_cast(routing[k * 2]) == expert_idx) { + float* output = activations.ffw_out.Row(bi); + const float* expert_out = activations.C1.Row(write_idx); + for (size_t d = 0; d < model_dim; ++d) output[d] += expert_out[d]; + ++write_idx; + break; + } + } + } + } // for each expert + + if (layer.ffn_output_w.HasPtr()) { + CallMatMul(activations.ffw_out, layer.ffn_output_w, + /*add=*/nullptr, env, activations.ffw_out); + } +} + + static inline void FFWNoVit(const LayerWeightsPtrs& layer, Activations& activations, MatMulEnv& env) { GCPP_ZONE(env.ctx, hwy::Profiler::GlobalIdx(), Zones::kGenFFW); @@ -159,9 +283,6 @@ static inline void FFWNoVit(const LayerWeightsPtrs& layer, HWY_DASSERT(!layer_config.ff_biases); // Only used in Vit. - activations.s_ffw_in.Notify(layer.layer_idx, activations.pre_ffw_rms_out, - env.ctx); - #if GEMMA_FUSED_FFN const auto fused = [&](RowPtrsBF C1, IndexRange range_r, IndexRange range_c, StridedViewBF C2, size_t worker) { @@ -183,31 +304,8 @@ static inline void FFWNoVit(const LayerWeightsPtrs& layer, env.ctx); #endif - activations.s_ffw_hidden.Notify(layer.layer_idx, activations.C1, env.ctx); - // Hidden layer -> output layer. CallMatMul(activations.C1, layer.linear_w, nullptr, env, activations.ffw_out); - - activations.s_ffw_out.Notify(layer.layer_idx, activations.ffw_out, env.ctx); -} - -// Sums encoded (`att_out`) over num_heads (`layer_config.heads`) and -// head_dim (`qkv_dim`) into output (`layer_out`). -static HWY_INLINE void SumHeads(const LayerWeightsPtrs& layer, - AttentionActivationsPtrs& activations, - MatMulEnv& env) { - GCPP_ZONE(env.ctx, hwy::Profiler::GlobalIdx(), Zones::kGenAttentionSumHeads); - const LayerConfig& layer_config = layer.layer_config; - (void)layer_config; // For HWY_DASSERT - // att_weights and att_out are concatenated heads, each of length - // layer_config.qkv_dim. Thus the [num_interleaved, - // layer_config.model_dim] matmul output is the sum over heads. Compare - // gemma/modules.py: attn_output = self.attn_vec_einsum('BTNH,NHD->BTD', - // encoded) - HWY_DASSERT(layer_config.model_dim != 0 && layer_config.heads != 0 && - layer_config.qkv_dim != 0); - CallMatMul(activations.att_out, layer.att_weights, /*add=*/nullptr, env, - activations.att_sums); } // NOLINTNEXTLINE(google-readability-namespace-comments) diff --git a/gemma/gemma.cc b/gemma/gemma.cc index ae94cc55..7c51655f 100644 --- a/gemma/gemma.cc +++ b/gemma/gemma.cc @@ -18,19 +18,12 @@ #include "gemma/gemma.h" -#include -#include -#include -#include - #include "compression/types.h" // GEMMA_DISABLED_TARGETS +#include "util/zones.h" #ifndef HWY_DISABLED_TARGETS #define HWY_DISABLED_TARGETS GEMMA_DISABLED_TARGETS #endif // HWY_DISABLED_TARGETS -#include "gemma/tensor_stats.h" -#include "util/zones.h" - // Compiles this file for multiple architectures via "foreach_target.h", to // which we pass the filename via macro 'argument'. // clang-format off @@ -42,8 +35,7 @@ // After highway.h #include "gemma/attention.h" // includes highway.h #include "gemma/gemma-inl.h" -#include "gemma/tiled_attention.h" // includes highway.h -#include "gemma/vit.h" // includes highway.h +#include "gemma/vit.h" // includes highway.h #ifndef GEMMA_CC_ONCE #define GEMMA_CC_ONCE @@ -81,20 +73,10 @@ namespace HWY_NAMESPACE { void Attention(LayerAttentionType type, const size_t num_tokens, const size_t layer_idx, const LayerWeightsPtrs& layer, Activations& activations, QBatch& qbatch, MatMulEnv& env) { - if (activations.attention_impl == AttentionImpl::kFlashTransposedQs || - activations.attention_impl == AttentionImpl::kFlashTransposedQsBF16 || - activations.attention_impl == AttentionImpl::kFlashTransposedQsInt16) { - TiledAttention( - activations.attention_impl, num_tokens, layer_idx, layer, - activations.attention, qbatch, env, - AttentionImplToFlags(activations.attention_impl, HWY_NATIVE_DOT_BF16)); - return; - } - if (type == LayerAttentionType::kGemma) { // TODO: remove flag to enable FlashAttention. GemmaAttention(num_tokens, layer_idx, layer, activations.attention, qbatch, - env, activations.attention_impl, /*flags=*/0); + env, HWY_NATIVE_DOT_BF16 ? kAttentionUseOld : 0); } } @@ -122,6 +104,8 @@ static HWY_NOINLINE void TransformerLayer(const size_t num_tokens, if (layer_config.type == LayerAttentionType::kVit) { FFWVit(layer, activations, env); + } else if (layer_config.num_experts != 0) { + FFWMoE(layer, activations, env); } else { FFWNoVit(layer, activations, env); } @@ -373,10 +357,6 @@ static HWY_NOINLINE void PrefillQBatch(const size_t max_prompt_size, (void)runtime_config.StreamToken(qbatch.QueryIdx(qi), pos_in_prompt, token, 0.0f); qbatch.MutablePos(qi) = pos_in_prompt; - } else { - // This prevents the kv cache of eos_id to be written to last prefilled - // token. - qbatch.MutablePos(qi) = qbatch.Prompt(qi).size(); } qbatch.PrevToken(qi) = token; @@ -442,13 +422,16 @@ static void SampleAndStream(const ModelConfig& config, } PROFILER_ZONE("Gen.Softcap+Sample+Stream"); - MaybeLogitsSoftCapBatched(config.final_cap, activations.logits, non_eos, - env.ctx); + // Use Gemma 4 soft capping if configured, otherwise fall back to final_cap. + const float softcap = (config.final_logit_softcapping > 0.0f) + ? config.final_logit_softcapping + : config.final_cap; + MaybeLogitsSoftCapBatched(softcap, activations.logits, non_eos, env.ctx); timing_info.NotifyGenerated(non_eos.Count()); ParallelFor( - Parallelism::kFlat, qbatch.Size(), env.ctx, + ParallelismStrategy::kFlat, qbatch.Size(), env.ctx, /*cluster_idx=*/0, Callers::kSampleAndStream, [&](size_t qi, size_t worker) { if (!non_eos.Get(qi)) return; @@ -506,11 +489,12 @@ ChooseSampleFunc(const RuntimeConfig& runtime_config, }; } -static size_t PrefillTBatchOrQBatch(const ModelConfig& config, - const RuntimeConfig& runtime_config, - const WeightsPtrs& weights, - Activations& activations, QBatch& qbatch, - MatMulEnv& env, TimingInfo& timing_info) { +// Decode: generates one continuation token for each query in `qbatch`. +static void GenerateT(const ModelConfig& config, + const RuntimeConfig& runtime_config, + const AesCtrEngine& engine, const WeightsPtrs& weights, + Activations& activations, QBatch& qbatch, MatMulEnv& env, + TimingInfo& timing_info) { size_t max_prompt_size = 0; bool all_prefix_end_are_zero = true; size_t total_prefill_tokens = 0; // only for throughput stats. @@ -532,14 +516,14 @@ static size_t PrefillTBatchOrQBatch(const ModelConfig& config, // We use a single divisor, so all sequence lengths must be the same. HWY_ASSERT(qbatch.KV(qi).SeqLen() == seq_len); } - if (max_prompt_size > seq_len) { - HWY_ABORT( - "max_prompt_size = %zu, seq_len = %zu, increase --seq_len to at least " - "that.", - max_prompt_size, seq_len); + if (max_prompt_size >= seq_len) { + HWY_ABORT("max_prompt_size = %zu, increase --seq_len to at least that.", + max_prompt_size); } HWY_ASSERT(activations.attention.div_seq_len.GetDivisor() == seq_len); + // Lacks a constructor to bulk-set, hence initialized by Prefill* which have + // qi loops anyway. hwy::BitSet4096<> non_eos; // indexed by qi timing_info.prefill_start = hwy::platform::Now(); @@ -557,6 +541,18 @@ static size_t PrefillTBatchOrQBatch(const ModelConfig& config, timing_info.NotifyPrefill(total_prefill_tokens); // queries_pos have been incremented by Prefill. + // Stream the last prompt token from each query, fill activations.gen_tokens. + for (size_t qi = 0; qi < qbatch.Size(); ++qi) { + const size_t last_pos_in_prompt = qbatch.Pos(qi) - qbatch.InitialPos(qi); + + const size_t pos = qbatch.Pos(qi); // during prefill, pos is still correct. + // In autoregressive mode, we have not prefilled the last token, so do + // not advance. + const bool update_pos = (qbatch.Pos(qi) < qbatch.PrefixEnd(qi)); + StreamAndUpdateEOS(qi, pos, qbatch.Prompt(qi)[last_pos_in_prompt], 0.0f, + config, runtime_config, qbatch, update_pos, non_eos); + } + size_t max_gen_steps = runtime_config.max_generated_tokens; if (max_prompt_size + max_gen_steps > seq_len) { HWY_WARN("prefill %zu + max_gen_steps %zu > seq_len %zu, truncating.", @@ -564,67 +560,9 @@ static size_t PrefillTBatchOrQBatch(const ModelConfig& config, max_gen_steps = seq_len - max_prompt_size; } - return max_gen_steps; -} - -static void StreamAndUpdateEOSAfterPrefill(const ModelConfig& config, - const RuntimeConfig& runtime_config, - QBatch& qbatch, - hwy::BitSet4096<>& non_eos, - size_t qi) { - const size_t last_pos_in_prompt = qbatch.Pos(qi) - qbatch.InitialPos(qi); - - const size_t pos = qbatch.Pos(qi); // during prefill, pos is still correct. - // In autoregressive mode, we have not prefilled the last token, so do - // not advance. - const bool update_pos = (qbatch.Pos(qi) < qbatch.PrefixEnd(qi)); - StreamAndUpdateEOS(qi, pos, qbatch.Prompt(qi)[last_pos_in_prompt], 0.0f, - config, runtime_config, qbatch, update_pos, non_eos); -} - -void SetWeightStats(const LayerWeightsPtrs& layer, Activations& a, - ThreadingContext& ctx) { - const size_t layer_idx = layer.layer_idx; - a.s_w_gating_einsum_w1.Notify(layer_idx, layer.gating_einsum_w1, ctx, - kTensorStatsIsWeight); - a.s_w_gating_einsum_w2.Notify(layer_idx, layer.gating_einsum_w2, ctx, - kTensorStatsIsWeight); - a.s_w_linear_w.Notify(layer_idx, layer.linear_w, ctx, kTensorStatsIsWeight); -} - -// Decode: generates one continuation token for each query in `qbatch`. -static void GenerateT(const ModelConfig& config, - const RuntimeConfig& runtime_config, - const AesCtrEngine& engine, const WeightsPtrs& weights, - Activations& activations, QBatch& qbatch, MatMulEnv& env, - TimingInfo& timing_info) { - for (const LayerWeightsPtrs& layer : weights.c_layers) { - SetWeightStats(layer, activations, env.ctx); - } - - MaybePrint(2, timing_info.verbosity, "[ BEGIN PHASE: prefill ]"); - const size_t max_gen_steps = PrefillTBatchOrQBatch( - config, runtime_config, weights, activations, qbatch, env, timing_info); - // No-op if the profiler is disabled, but useful to separate prefill and - // generate phases for profiling. - if constexpr (PROFILER_ENABLED) { - fprintf(stderr, "\n"); - } - env.ctx.profiler.PrintResults(); - - hwy::BitSet4096<> non_eos; // indexed by qi - - // Stream the last prompt token from each query, fill activations.gen_tokens. - for (size_t qi = 0; qi < qbatch.Size(); ++qi) { - non_eos.Set(qi); - StreamAndUpdateEOSAfterPrefill(config, runtime_config, qbatch, non_eos, qi); - } - const SampleFunc sample_token = ChooseSampleFunc(runtime_config, engine, env.ctx); - MaybePrint(2, timing_info.verbosity, "\n[ BEGIN PHASE: generate ]\n"); - timing_info.generate_start = hwy::platform::Now(); for (size_t gen = 0; gen < max_gen_steps && non_eos.Any(); ++gen) { Transformer(config, runtime_config, weights, activations, qbatch, env); @@ -634,66 +572,14 @@ static void GenerateT(const ModelConfig& config, timing_info.NotifyGenerateDone(); } -// Same as GenerateT, but uses ContinuousQBatch. -static void GenerateTWithContinuousBatching( - const ModelConfig& config, const RuntimeConfig& runtime_config, - const AesCtrEngine& engine, const WeightsPtrs& weights, - Activations& activations, AllQueries& all_queries, MatMulEnv& env, - TimingInfo& timing_info) { - const size_t qbatch_size = runtime_config.decode_qbatch_size; - - QBatch qbatch(0, qbatch_size, all_queries); - ContinuousQBatch prefill_batch(qbatch_size, all_queries); - - hwy::BitSet4096<> non_eos; - const SampleFunc sample_token = - ChooseSampleFunc(runtime_config, engine, env.ctx); - - size_t query_inserted = 0; - while (non_eos.Any() || query_inserted < all_queries.NumQueries()) { - for (size_t qi = 0; qi < qbatch.Size(); ++qi) { - // Continue if qi slot is still processing. - if (non_eos.Get(qi)) continue; - // Collect the kv_cache from the qi slot in the qbatch to the - // available_kv_caches_ in the prefill_batch. - prefill_batch.MaybeReleaseKV(qbatch.Single(qi)); - - // Prefill if no available prefilled queries to insert. - if (prefill_batch.ShouldPrefill()) { - prefill_batch.SetupNextBatchForPrefill(); - PrefillTBatchOrQBatch(config, runtime_config, weights, activations, - prefill_batch, env, timing_info); - activations.SetBatchSize(qbatch.Size()); - } - - // Get the next query to insert to the generate batch. - std::optional qi_to_insert = prefill_batch.GetNextToInsert(); - if (qi_to_insert) { - qbatch.Insert(qi_to_insert.value(), qi); - query_inserted++; - - non_eos.Set(qi); - StreamAndUpdateEOSAfterPrefill(config, runtime_config, qbatch, non_eos, - qi); - } - } - - Transformer(config, runtime_config, weights, activations, qbatch, env); - SampleAndStream(config, runtime_config, weights, sample_token, activations, - qbatch, env, non_eos, timing_info); - } - timing_info.NotifyGenerateDone(); -} - void GenerateSingleT(const PromptTokens& prompt, size_t pos, size_t prefix_end, const ModelConfig& config, const RuntimeConfig& runtime_config, const AesCtrEngine& engine, const WeightsPtrs& weights, KVCache& kv_cache, MatMulEnv& env, TimingInfo& timing_info) { - Activations activations(runtime_config, config, - runtime_config.prefill_tbatch_size, kv_cache.SeqLen(), - env.ctx, env.row_ptrs); + Activations activations(config, runtime_config.prefill_tbatch_size, + kv_cache.SeqLen(), env.ctx, env.row_ptrs); AllQueries all_queries(prompt, pos, prefix_end, hwy::Span(&kv_cache, 1)); @@ -711,55 +597,36 @@ void GenerateBatchT(const ModelConfig& config, TimingInfo& timing_info) { const size_t max_batch_size = HWY_MAX(runtime_config.decode_qbatch_size, runtime_config.prefill_tbatch_size); - Activations activations(runtime_config, config, max_batch_size, + Activations activations(config, max_batch_size, all_queries[0].kv_cache.SeqLen(), env.ctx, env.row_ptrs); - if (runtime_config.use_continuous_batching) { - GenerateTWithContinuousBatching(config, runtime_config, engine, weights, - activations, all_queries, env, timing_info); - } else { - for (size_t start = 0; start < all_queries.NumQueries(); - start += runtime_config.decode_qbatch_size) { - QBatch qbatch(start, runtime_config.decode_qbatch_size, all_queries); - // Generate a batch of one token for each of `qbatch.Size()` queries. - GenerateT(config, runtime_config, engine, weights, activations, qbatch, - env, timing_info); - } + for (size_t start = 0; start < all_queries.NumQueries(); + start += runtime_config.decode_qbatch_size) { + QBatch qbatch(start, runtime_config.decode_qbatch_size, all_queries); + // Generate a batch of one token for each of `qbatch.Size()` queries. + GenerateT(config, runtime_config, engine, weights, activations, qbatch, env, + timing_info); } } void GenerateImageTokensT(const ModelConfig& config, const RuntimeConfig& runtime_config, size_t seq_len, const WeightsPtrs& weights, const Image& image, - ImageTokens& image_tokens, MatMulEnv& env, - TimingInfo& timing_info) { + ImageTokens& image_tokens, MatMulEnv& env) { + if (config.vit_config.layer_configs.empty()) { + HWY_ABORT("Model does not support generating image tokens."); + } + RuntimeConfig prefill_runtime_config = runtime_config; const ModelConfig vit_config = GetVitConfig(config); const size_t num_tokens = vit_config.max_seq_len; - - MaybePrint(2, timing_info.verbosity, "\n[ BEGIN PHASE: image_token_gen ]\n"); - timing_info.NotifyImageTokenStart(); - - { - GCPP_ZONE(env.ctx, hwy::Profiler::GlobalIdx(), Zones::kGenImageTokens); - if (config.vit_config.layer_configs.empty()) { - HWY_ABORT("Model does not support generating image tokens."); - } - RuntimeConfig prefill_runtime_config = runtime_config; - prefill_runtime_config.prefill_tbatch_size = - num_tokens / (vit_config.pool_dim * vit_config.pool_dim); - Activations prefill_activations(runtime_config, vit_config, num_tokens, - num_tokens, env.ctx, env.row_ptrs); - // Weights are for the full PaliGemma model, not just the ViT part. - PrefillVit(config, weights, prefill_runtime_config, image, image_tokens, - prefill_activations, env); - } // end GCPP_ZONE before we print results. - - // No-op if the profiler is disabled. Printing now ensures that the - // `PrintResults` after prefill does not include the image token part. - env.ctx.profiler.PrintResults(); - - timing_info.NotifyImageTokenDone(num_tokens); + prefill_runtime_config.prefill_tbatch_size = + num_tokens / (vit_config.pool_dim * vit_config.pool_dim); + Activations prefill_activations(vit_config, num_tokens, num_tokens, env.ctx, + env.row_ptrs); + // Weights are for the full PaliGemma model, not just the ViT part. + PrefillVit(config, weights, prefill_runtime_config, image, image_tokens, + prefill_activations, env); } // NOLINTNEXTLINE(google-readability-namespace-comments) @@ -773,22 +640,17 @@ HWY_EXPORT(GenerateSingleT); HWY_EXPORT(GenerateBatchT); HWY_EXPORT(GenerateImageTokensT); -Gemma::Gemma(const GemmaArgs& args, ThreadingContext& ctx) - : reader_(args.loader.weights), - model_(reader_, args.loader.tokenizer, args.loader.wrapping), +Gemma::Gemma(const LoaderArgs& loader, const InferenceArgs& inference, + ThreadingContext& ctx) + : reader_(loader.weights), + model_(reader_, loader.tokenizer, loader.wrapping), weights_(model_.Config()), chat_template_(model_.Tokenizer(), model_.Config().model), - inference_(args.inference), - aes_ctr_engine_(args.inference.deterministic) { - if (args.inference.seq_len > model_.Config().max_seq_len) { - HWY_WARN( - "Overriding model's max_seq_len=%u with user provided seq_len=%zu.", - model_.Config().max_seq_len, args.inference.seq_len); - model_.MutableConfig().SetMaxSeqLen(args.inference.seq_len); - } + inference_(inference), + aes_ctr_engine_(inference.deterministic) { // Negligible CPU time in the ctor body (except ReadFromBlobs). - weight_read_mode_ = weights_.ReadFromBlobs(model_, reader_, args.loader, - args.inference, mat_owners_, ctx); + weight_read_mode_ = weights_.ReadFromBlobs(model_, reader_, loader, inference, + mat_owners_, ctx); // Read everything into memory, or `weights_.mapped_` keeps the mapping alive. reader_.CloseFile(); } @@ -830,75 +692,16 @@ void Gemma::GenerateBatch(const RuntimeConfig& runtime_config, void Gemma::GenerateImageTokens(const RuntimeConfig& runtime_config, size_t seq_len, const Image& image, - ImageTokens& image_tokens, MatMulEnv& env, - TimingInfo& timing_info) const { + ImageTokens& image_tokens, + MatMulEnv& env) const { env.ctx.pools.MaybeStartSpinning(runtime_config.use_spinning); HWY_DYNAMIC_DISPATCH(GenerateImageTokensT)(model_.Config(), runtime_config, seq_len, weights_, image, - image_tokens, env, timing_info); + image_tokens, env); env.ctx.pools.MaybeStopSpinning(runtime_config.use_spinning); } -ContinuousQBatch::ContinuousQBatch(size_t max_size, AllQueries& queries) - : QBatch(0, max_size, queries) { - for (size_t i = start_; i < queries_.NumQueries(); ++i) { - if (!queries_[i].kv_cache.IsEmpty()) { - // Put the kv_cache to the available_kv_caches_ instead; leaving the - // kv_cache in the queries_ is very confusing. This simplifies the logic - // of kv_cache management. - available_kv_caches_.push_back(queries_[i].kv_cache); - queries_[i].kv_cache = KVCachePtr(); - } - } -} - -bool ContinuousQBatch::ShouldPrefill() const { - const bool no_available_to_insert = next_to_insert_ == next_to_prefill_; - const int more_queries_to_prefill = next_to_prefill_ < queries_.NumQueries(); - return no_available_to_insert && more_queries_to_prefill; -} - -void ContinuousQBatch::SetupNextBatchForPrefill() { - start_ = next_to_prefill_; - size_ = HWY_MIN(max_size_, queries_.NumQueries() - start_); - HWY_DASSERT(size_ != 0); - HWY_DASSERT(start_ + size_ <= queries_.NumQueries()); - query_idx_.clear(); - query_idx_.reserve(size_); - for (size_t i = 0; i < size_; ++i) { - const size_t next_query_idx = start_ + i; - query_idx_.push_back(next_query_idx); - HWY_ASSERT(queries_[next_query_idx].kv_cache.IsEmpty()); - queries_[next_query_idx].kv_cache = available_kv_caches_.back(); - available_kv_caches_.pop_back(); - } - next_to_prefill_ += size_; -} - -std::optional ContinuousQBatch::GetNextToInsert() { - if (next_to_insert_ == next_to_prefill_) { - return std::nullopt; - } - next_to_insert_++; - return next_to_insert_ - 1; -} - -void ContinuousQBatch::MaybeReleaseKV(const QBatch& from) { - const int query_to_collect = from.QueryIdx(0); - // Only collect if the query to collect is not the same as the next query to - // insert. This happens at the beginning of each Generate call. - if (query_to_collect != next_to_insert_) { - // Only clear the KV cache if there are more queries to insert; Otherwise - // we get a crash because Transformer will still access that KV cache. - if (next_to_insert_ < queries_.NumQueries()) { - available_kv_caches_.push_back(from.KV(0)); - ZeroInit(from.KV(0).kv_cache); - from.KV(0) = KVCachePtr(); - } - } -} - } // namespace gcpp #endif // HWY_ONCE diff --git a/gemma/tensor_info.cc b/gemma/tensor_info.cc index 2810307a..71649f36 100644 --- a/gemma/tensor_info.cc +++ b/gemma/tensor_info.cc @@ -1,18 +1,3 @@ -// Copyright 2025 Google LLC -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - #include "gemma/tensor_info.h" #include @@ -452,6 +437,53 @@ void TensorInfoRegistry::AddLayerTensors(const ModelConfig& config, .shape = {config.model_dim, layer_config.heads, layer_config.qkv_dim}, .cols_take_extra_dims = true, }); + + // Gemma 4 MoE weights. + if (layer_config.num_experts != 0) { + Add(suffix, { + .base_name = "ffn_gate_in_w", + .source_names = {"mlp/gate_in/kernel"}, + .axes = {1, 0}, + .shape = {config.model_dim, layer_config.num_experts}, + }); + Add(suffix, { + .base_name = "ffn_gate_w", + .source_names = {"mlp/gate/kernel"}, + .axes = {0, 2, 1}, + .shape = {layer_config.num_experts, + layer_config.moe_intermediate_size, + config.model_dim}, + }); + Add(suffix, { + .base_name = "ffn_up_w", + .source_names = {"mlp/up/kernel"}, + .axes = {0, 2, 1}, + .shape = {layer_config.num_experts, + layer_config.moe_intermediate_size, + config.model_dim}, + }); + Add(suffix, { + .base_name = "ffn_down_w", + .source_names = {"mlp/down/kernel"}, + .axes = {0, 1, 2}, + .shape = {layer_config.num_experts, + config.model_dim, + layer_config.moe_intermediate_size}, + }); + Add(suffix, { + .base_name = "ffn_output_w", + .source_names = {"mlp/output/kernel"}, + .axes = {1, 0}, + .shape = {config.model_dim, config.model_dim}, + }); + } + Add(suffix, { + .base_name = "attn_kv_scale", + .source_names = {"attn/kv_scale"}, + .axes = {0}, + .shape = {1}, + .min_size = Type::kF32, + }); } TensorInfoRegistry::TensorInfoRegistry(const ModelConfig& config) { diff --git a/gemma/weights.h b/gemma/weights.h index 4476e22d..2603c708 100644 --- a/gemma/weights.h +++ b/gemma/weights.h @@ -96,8 +96,7 @@ struct LayerWeightsPtrs { // other values for purposes of the KV cache. LayerWeightsPtrs(size_t layer_idx, const LayerConfig& config, const TensorInfoRegistry& tensors) - : layer_idx(layer_idx), - finder_(LayerSuffix(layer_idx), tensors), + : finder_(LayerSuffix(layer_idx), tensors), qkv_einsum_w(finder_("qkv_ein")), qkv_einsum_w1(finder_("qkv1_w")), qkv_einsum_w2(finder_("qkv2_w")), @@ -131,12 +130,18 @@ struct LayerWeightsPtrs { key_norm_scale(finder_("key_norm")), query_norm_scale(finder_("query_norm")), + // Gemma 4 MoE weights. + ffn_gate_in_w(finder_("ffn_gate_in_w")), + ffn_gate_w(finder_("ffn_gate_w")), + ffn_up_w(finder_("ffn_up_w")), + ffn_down_w(finder_("ffn_down_w")), + ffn_output_w(finder_("ffn_output_w")), + attn_kv_scale(finder_("attn_kv_scale")), layer_config(config) { } ~LayerWeightsPtrs() = default; - const size_t layer_idx; const MatFinder finder_; // Files either have qkv_einsum_w with 2 stacked matrices or separate @@ -184,6 +189,14 @@ struct LayerWeightsPtrs { MatPtr key_norm_scale; // at least BF16. MatPtr query_norm_scale; // at least BF16. + // Gemma 4 MoE weights. + MatPtr ffn_gate_in_w; // Router weights. + MatPtr ffn_gate_w; // Expert gate (up proj). + MatPtr ffn_up_w; // Expert up proj. + MatPtr ffn_down_w; // Expert down proj. + MatPtr ffn_output_w; // MoE output. + MatPtrT attn_kv_scale; // Per-layer KV scaling. + const LayerConfig& layer_config; // Calls `func(TensorArgs)` for each tensor which is in use for the @@ -243,6 +256,15 @@ struct LayerWeightsPtrs { func(TENSOR_ARGS(ffw_gating_biases, kMustRead)); func(TENSOR_ARGS(ffw_output_biases, kMustRead)); } + // Gemma 4 MoE weights. + if (layer_config.num_experts != 0) { + func(TENSOR_ARGS(ffn_gate_in_w, kMaybeRead)); + func(TENSOR_ARGS(ffn_gate_w, kMaybeRead)); + func(TENSOR_ARGS(ffn_up_w, kMaybeRead)); + func(TENSOR_ARGS(ffn_down_w, kMaybeRead)); + func(TENSOR_ARGS(ffn_output_w, kMaybeRead)); + } + func(TENSOR_ARGS(attn_kv_scale, kMaybeRead)); } // `ForEachTensor` // Zero-initializes all allocated tensors in the layer.