diff --git a/gemma/activations.h b/gemma/activations.h index 86b2ae27..e68c71d7 100644 --- a/gemma/activations.h +++ b/gemma/activations.h @@ -100,11 +100,17 @@ struct AttentionActivations { vit_Q(MatFactory("Q2", batch_size, max_qkv_dim, allocator)), vit_K_T(MatFactory( - "K2_T", hwy::RoundUpTo(seq_len, kMaxBF16PerVector), + "K2_T", + config.vocab_size == 0 + ? hwy::RoundUpTo(seq_len, kMaxBF16PerVector) + : 0, layer_config.heads * hwy::RoundUpTo(max_qkv_dim, kMaxBF16PerVector), allocator, MatPadding::kPacked)), vit_V_T(MatFactory( - "V2_T", hwy::RoundUpTo(seq_len, kMaxBF16PerVector), + "V2_T", + config.vocab_size == 0 + ? hwy::RoundUpTo(seq_len, kMaxBF16PerVector) + : 0, layer_config.heads * hwy::RoundUpTo(max_qkv_dim, kMaxBF16PerVector), allocator, MatPadding::kPacked)), pre_att_rms_out(MatFactory("pre_att_rms_out", batch_size, diff --git a/gemma/configs.h b/gemma/configs.h index f1f63510..15c55692 100644 --- a/gemma/configs.h +++ b/gemma/configs.h @@ -365,7 +365,10 @@ void ForEachModel(const Func& func) { } } -static inline bool& IsInternalActive() { +#if defined(__GNUC__) || defined(__clang__) +__attribute__((visibility("default"))) +#endif +inline bool& IsInternalActive() { static bool active = false; return active; } diff --git a/gemma/flash_attention.cc b/gemma/flash_attention.cc index 5092f514..12641d6a 100644 --- a/gemma/flash_attention.cc +++ b/gemma/flash_attention.cc @@ -1886,10 +1886,11 @@ void DispatchTileFlashAttentionReturnExpSumsAndMaxLogitsAMX( hwy::Span start_pos_per_query, hwy::Span last_pos_per_query, const float att_cap, MatPtrT& att_out, float* HWY_RESTRICT exp_denominator_sums, - float* HWY_RESTRICT max_logits) { + float* HWY_RESTRICT max_logits, + hwy::AlignedVector* worker_workspace) { TileFlashAttentionReturnExpSumsAndMaxLogitsAMX( kvs, q_count, q_base, start_pos_per_query, last_pos_per_query, att_cap, - att_out, exp_denominator_sums, max_logits); + att_out, exp_denominator_sums, max_logits, worker_workspace); } void DispatchTileFlashAttentionReturnExpSumsAndMaxLogitsAMXInt8( diff --git a/gemma/flash_attention.h b/gemma/flash_attention.h index c958aa47..23c9e7a2 100644 --- a/gemma/flash_attention.h +++ b/gemma/flash_attention.h @@ -75,7 +75,8 @@ namespace gcpp { hwy::Span start_pos_per_query, \ hwy::Span last_pos_per_query, const float att_cap, \ MatPtrT& att_out, float* HWY_RESTRICT exp_denominator_sums, \ - float* HWY_RESTRICT max_logits); \ + float* HWY_RESTRICT max_logits, \ + hwy::AlignedVector* worker_workspace); \ \ void DispatchTileFlashAttentionReturnExpSumsAndMaxLogitsAMXInt8( \ hwy::Span kvs, size_t q_count, \ diff --git a/gemma/flash_attention_amx-inl.h b/gemma/flash_attention_amx-inl.h index 644cb27e..1394d1e0 100644 --- a/gemma/flash_attention_amx-inl.h +++ b/gemma/flash_attention_amx-inl.h @@ -1392,7 +1392,8 @@ inline void TileFlashAttentionReturnExpSumsAndMaxLogitsAMX( hwy::Span start_pos_per_query, hwy::Span last_pos_per_query, const float att_cap, MatPtrT& att_out, float* HWY_RESTRICT exp_denominator_sums, - float* HWY_RESTRICT max_logits) { + float* HWY_RESTRICT max_logits, + hwy::AlignedVector* worker_workspace = nullptr) { #if GEMMA_HAVE_AMX if (hwy::HaveTile64BMatMulBF16()) { TileFlashAttentionAMX_BF16_Impl(kvs, q_count, q_base, start_pos_per_query, @@ -1403,8 +1404,7 @@ inline void TileFlashAttentionReturnExpSumsAndMaxLogitsAMX( #endif // GEMMA_HAVE_AMX DispatchTileFlashAttentionReturnExpSumsAndMaxLogitsBF16( kvs, q_count, q_base, start_pos_per_query, last_pos_per_query, att_cap, - att_out, exp_denominator_sums, max_logits, - /*worker_workspace=*/nullptr); + att_out, exp_denominator_sums, max_logits, worker_workspace); } inline void TileFlashAttentionReturnExpSumsAndMaxLogitsAMXInt8( diff --git a/gemma/flash_attention_test.cc b/gemma/flash_attention_test.cc index 88649b74..48a79001 100644 --- a/gemma/flash_attention_test.cc +++ b/gemma/flash_attention_test.cc @@ -700,7 +700,8 @@ void RunTiledFlashAttentionTest(gcpp::KVEncoding kv_encoding, kvs, num_queries, bf16_queries.data(), hwy::Span(start_pos_per_query), hwy::Span(last_pos_per_query), att_cap, att_out, - exp_denominator_sums.data(), max_logits.data()); + exp_denominator_sums.data(), max_logits.data(), + /*worker_workspace=*/nullptr); } else if (attention_impl == AttentionImpl::kFlashAMXInt8) { std::vector> int8_queries( num_queries * qkv_dim); @@ -998,7 +999,7 @@ struct TiledFlashAttentionTestSetup { case TiledTestKernel::kAmxBF16: DispatchTileFlashAttentionReturnExpSumsAndMaxLogitsAMX( kvs, q_count, q_bf16.data() + first_q * kQkvDim, starts, lasts, - /*att_cap=*/0.0f, att_out, exp_sums, max_logits); + /*att_cap=*/0.0f, att_out, exp_sums, max_logits, workspace); break; case TiledTestKernel::kAmxInt8: DispatchTileFlashAttentionReturnExpSumsAndMaxLogitsAMXInt8( @@ -1453,7 +1454,8 @@ void RunTiledFlashAttentionDifferentialTest( kvs, num_queries, bf16_queries.data(), hwy::Span(start_pos_per_query), hwy::Span(last_pos_per_query), att_cap, att_out, - exp_denominator_sums.data(), max_logits.data()); + exp_denominator_sums.data(), max_logits.data(), + /*worker_workspace=*/nullptr); } else if (opt_impl == AttentionImpl::kFlashAMXInt8) { DispatchTileFlashAttentionReturnExpSumsAndMaxLogitsAMXInt8( kvs, num_queries, int8_queries.data(), q_scales, diff --git a/gemma/kv_cache.cc b/gemma/kv_cache.cc index e83f9ed4..1f130912 100644 --- a/gemma/kv_cache.cc +++ b/gemma/kv_cache.cc @@ -280,34 +280,46 @@ KVCache::KVCache(const ModelConfig& config, const InferenceArgs& inference_args, if (runtime_config.attention_impl == AttentionImpl::kFlash || IsTiledAttention(runtime_config.attention_impl)) { + const bool tiled_only = IsTiledAttention(runtime_config.attention_impl) && + !config.is_encoder_decoder && !config.HasMLA() && + config.num_mtp_layers == 0; kv_is_scratch = runtime_config.attention_impl == AttentionImpl::kFlash && !config.is_encoder_decoder && !config.HasMLA(); - size_t projection_cols = config.KVCacheCols(); - size_t projection_rows = CappedSeqLen(config, inference_args); - if (kv_is_scratch) { - projection_cols = 0; - for (const LayerConfig& layer : kv_layer_configs) { - projection_cols = std::max(projection_cols, layer.CacheLayerSize()); + if (tiled_only) { + // TiledAttention exclusively reads and writes compact_local_kv_cache_ptr + // and compact_global_kv_cache_ptr. Keep a 1x1 sentinel in kv_cache so + // KVCachePtr::IsEmpty() returns false without allocating O(seq_len) + // unused flat/transposed buffers (36 GiB saved at 1M context). + kv_cache = MatStorageT("kv", Extents2D(1, 1), allocator, + MatPadding::kOdd); + } else { + size_t projection_cols = config.KVCacheCols(); + size_t projection_rows = CappedSeqLen(config, inference_args); + if (kv_is_scratch) { + projection_cols = 0; + for (const LayerConfig& layer : kv_layer_configs) { + projection_cols = std::max(projection_cols, layer.CacheLayerSize()); + } + projection_rows = + std::min(projection_rows, + std::max(size_t{1}, runtime_config.prefill_tbatch_size)); } - projection_rows = - std::min(projection_rows, - std::max(size_t{1}, runtime_config.prefill_tbatch_size)); + kv_cache = + MatStorageT("kv", Extents2D(projection_rows, projection_cols), + allocator, MatPadding::kOdd); + k_cache = MatStorageT( + "k", + Extents2D(hwy::RoundUpTo(CappedSeqLen(config, inference_args), + kMaxBF16PerVector), + k_v_cols), + allocator, MatPadding::kPacked); + v_cache = MatStorageT( + "v", + Extents2D(hwy::RoundUpTo(CappedSeqLen(config, inference_args), + kMaxBF16PerVector), + k_v_cols), + allocator, MatPadding::kPacked); } - kv_cache = - MatStorageT("kv", Extents2D(projection_rows, projection_cols), - allocator, MatPadding::kOdd); - k_cache = MatStorageT( - "k", - Extents2D(hwy::RoundUpTo(CappedSeqLen(config, inference_args), - kMaxBF16PerVector), - k_v_cols), - allocator, MatPadding::kPacked); - v_cache = MatStorageT( - "v", - Extents2D(hwy::RoundUpTo(CappedSeqLen(config, inference_args), - kMaxBF16PerVector), - k_v_cols), - allocator, MatPadding::kPacked); const size_t num_tiles = hwy::DivCeil(CappedSeqLen(config, inference_args), kTileSize); tiled_seq_len = num_tiles * kTileSize; @@ -489,6 +501,58 @@ void KVCache::ZeroInit() { if (ds_state_snapshot.HasPtr()) gcpp::ZeroInit(ds_state_snapshot); } +void KVCache::ZeroInitPrefix(size_t dirty_tokens) { + if (dirty_tokens == 0) return; + + if (!kv_head_ptrs.empty()) { + const size_t needed_tiles = hwy::DivCeil(dirty_tokens, kTileSize) + 1; + for (MatPtr& hp : kv_head_ptrs) { + if (!hp.HasPtr()) continue; + const size_t dirty_tiles = std::min(hp.Rows(), needed_tiles); + hwy::ZeroBytes(hp.RowBytes(0), + dirty_tiles * hp.Stride() * hp.ElementBytes()); + } + } else { + if (compact_local_kv_cache_ptr.HasPtr()) { + gcpp::ZeroInit(compact_local_kv_cache_ptr); + } + if (compact_global_kv_cache_ptr.HasPtr()) { + gcpp::ZeroInit(compact_global_kv_cache_ptr); + } + } + + if (kv_cache.HasPtr()) { + if (kv_is_scratch || kv_cache.Rows() <= dirty_tokens) { + gcpp::ZeroInit(kv_cache); + } else { + hwy::ZeroBytes(kv_cache.RowBytes(0), + dirty_tokens * kv_cache.Stride() * + kv_cache.ElementBytes()); + } + } + + auto zero_kv_prefix = [&](MatStorageT& cache) { + if (!cache.HasPtr()) return; + size_t dirty_rows = cache.Rows(); + if (k_v_cols > 0 && cache.Cols() > k_v_cols) { + const size_t group = cache.Cols() / k_v_cols; + dirty_rows = + std::min(cache.Rows(), hwy::DivCeil(dirty_tokens, group) + 1); + } else { + dirty_rows = std::min( + cache.Rows(), + hwy::RoundUpTo(dirty_tokens, kMaxBF16PerVector) + kMaxBF16PerVector); + } + hwy::ZeroBytes(cache.RowBytes(0), + dirty_rows * cache.Stride() * cache.ElementBytes()); + }; + zero_kv_prefix(k_cache); + zero_kv_prefix(v_cache); + + if (ds_state.HasPtr()) gcpp::ZeroInit(ds_state); + if (ds_state_snapshot.HasPtr()) gcpp::ZeroInit(ds_state_snapshot); +} + void KVCachePtr::ZeroInit() { if (cache != nullptr) { cache->ZeroInit(); diff --git a/gemma/kv_cache.h b/gemma/kv_cache.h index e4132577..c10089d3 100644 --- a/gemma/kv_cache.h +++ b/gemma/kv_cache.h @@ -56,6 +56,9 @@ struct KVCache { // copy ctor to make the cost explicit. KVCache Copy(); void ZeroInit(); + // Zeroes only KV cache storage that could have been dirtied by evaluating up + // to `dirty_tokens` positions starting from position 0. + void ZeroInitPrefix(size_t dirty_tokens); // Default flash only needs the current layer's projections until transpose. // Other backends can still use kv_cache as persistent inference state. diff --git a/gemma/tiled_attention.cc b/gemma/tiled_attention.cc index a84131b2..4dcf2f5d 100644 --- a/gemma/tiled_attention.cc +++ b/gemma/tiled_attention.cc @@ -1480,7 +1480,7 @@ void LocalAttentionForAllHeadsTokensAndBatch( hwy::Span(start_pos_per_query), hwy::Span(last_pos_per_query), activations.config.att_cap, att_out, exp_denominator_sums.data(), - max_logits.data()); + max_logits.data(), worker_workspace); } else if (attention_impl == AttentionImpl::kFlashTransposedQsInt16) { HWY_DASSERT(activations.int16_queries != nullptr); diff --git a/python/gemma_py.cc b/python/gemma_py.cc index ad820900..59760488 100644 --- a/python/gemma_py.cc +++ b/python/gemma_py.cc @@ -21,7 +21,11 @@ #include #include +#include +#include +#include #include +#include #include #include #include @@ -47,7 +51,11 @@ static void RemoveTrailingZeros(std::vector &vec) { // Wrapper around GemmaEnv to expose to Python. class GemmaModel { public: - GemmaModel(const gcpp::GemmaArgs& args) : env_(args), last_prob_(0.0f) {} + GemmaModel(const gcpp::GemmaArgs& args, size_t max_seq_len = 0) + : env_(args), + last_prob_(0.0f), + max_seq_len_(max_seq_len > 0 ? max_seq_len + : env_.MutableKVCache().SeqLen()) {} // Generates a single example, given a prompt and a callback to stream the // generated tokens. @@ -282,6 +290,10 @@ class GemmaModel { float GetLastProb() const { return last_prob_; } + std::vector Tokenize(const std::string& text) const { + return env_.Tokenize(text); + } + std::string Detokenize(const std::vector& token_ids) const { return env_.StringFromTokens(token_ids); } @@ -289,32 +301,65 @@ class GemmaModel { bool ModelIsLoaded() const { return env_.GetGemma() != nullptr; } private: + gcpp::GemmaEnv env_; std::unique_ptr image_tokens_; + std::optional eval_kv_cache_; + std::unique_ptr eval_activations_; + size_t eval_activations_max_tbatch_ = 0; + size_t eval_activations_kv_seq_len_ = 0; + gcpp::AttentionImpl eval_activations_att_impl_ = gcpp::AttentionImpl::kFlash; + size_t dirty_kv_tokens_ = std::numeric_limits::max(); gcpp::TimingInfo timing_info_; float last_prob_; + size_t max_seq_len_ = 0; }; PYBIND11_MODULE(gemma, mod) { - py::class_(mod, "GemmaModel") + py::class_ gemma_cls(mod, "GemmaModel"); + gemma_cls .def(py::init([](const std::string& tokenizer, const std::string& weights, - size_t max_threads) { + size_t max_threads, size_t seq_len, + size_t prefill_tbatch_size, + const std::string& attention_impl, bool map_weights) { gcpp::ThreadingArgs threading; threading.max_lps = max_threads; gcpp::InferenceArgs inference; inference.max_generated_tokens = 512; + if (seq_len > 0) { + // Reserve one extra SIMD row tile (kMaxBF16PerVector) in the + // initial KVCache so AVX-512 batched prefill does not need to + // allocate a second KVCache in ComputeLikelihood. + inference.seq_len = + hwy::RoundUpTo(seq_len, gcpp::kMaxBF16PerVector) + + gcpp::kMaxBF16PerVector; + } + if (prefill_tbatch_size > 0) { + inference.prefill_tbatch_size = prefill_tbatch_size; + } + if (attention_impl == "auto") { + inference.attention_impl = "flash_amx"; + } else if (!attention_impl.empty()) { + inference.attention_impl = attention_impl; + } - const gcpp::GemmaArgs args(gcpp::LoaderArgs(tokenizer, weights), - threading, inference); - auto gemma = std::make_unique(args); + gcpp::LoaderArgs loader_args(tokenizer, weights); + if (map_weights) { + loader_args.map = gcpp::Tristate::kTrue; + } + const gcpp::GemmaArgs args(loader_args, threading, inference); + auto gemma = std::make_unique(args, seq_len); if (!gemma->ModelIsLoaded()) { throw std::invalid_argument("Could not load model."); } return gemma; }), py::arg("tokenizer_path"), py::arg("weights_path"), - py::arg("max_threads") = 0) + py::arg("max_threads") = 0, py::arg("seq_len") = 0, + py::arg("prefill_tbatch_size") = 0, + py::arg("attention_impl") = "flash", + py::arg("map_weights") = false) // seed arguments are ignored. .def("generate_ex", &GemmaModel::GenerateEx, py::arg("prompt"), py::arg("stream"), py::arg("max_generated_tokens") = 1024, @@ -338,5 +383,6 @@ PYBIND11_MODULE(gemma, mod) { .def("score_next", &GemmaModel::ScoreNext, py::arg("prompt"), py::arg("candidates")) .def("get_last_prob", &GemmaModel::GetLastProb) + .def("tokenize", &GemmaModel::Tokenize, py::arg("text")) .def("detokenize", &GemmaModel::Detokenize, py::arg("token_ids")); }