Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions gemma/activations.h
Original file line number Diff line number Diff line change
Expand Up @@ -100,11 +100,17 @@ struct AttentionActivations {

vit_Q(MatFactory("Q2", batch_size, max_qkv_dim, allocator)),
vit_K_T(MatFactory(
"K2_T", hwy::RoundUpTo(seq_len, kMaxBF16PerVector),
"K2_T",
config.vocab_size == 0
? hwy::RoundUpTo(seq_len, kMaxBF16PerVector)
: 0,
layer_config.heads * hwy::RoundUpTo(max_qkv_dim, kMaxBF16PerVector),
allocator, MatPadding::kPacked)),
vit_V_T(MatFactory(
"V2_T", hwy::RoundUpTo(seq_len, kMaxBF16PerVector),
"V2_T",
config.vocab_size == 0
? hwy::RoundUpTo(seq_len, kMaxBF16PerVector)
: 0,
layer_config.heads * hwy::RoundUpTo(max_qkv_dim, kMaxBF16PerVector),
allocator, MatPadding::kPacked)),
pre_att_rms_out(MatFactory("pre_att_rms_out", batch_size,
Expand Down
5 changes: 4 additions & 1 deletion gemma/configs.h
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,10 @@ void ForEachModel(const Func& func) {
}
}

static inline bool& IsInternalActive() {
#if defined(__GNUC__) || defined(__clang__)
__attribute__((visibility("default")))
#endif
inline bool& IsInternalActive() {
static bool active = false;
return active;
}
Expand Down
5 changes: 3 additions & 2 deletions gemma/flash_attention.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1886,10 +1886,11 @@ void DispatchTileFlashAttentionReturnExpSumsAndMaxLogitsAMX(
hwy::Span<const size_t> start_pos_per_query,
hwy::Span<const size_t> last_pos_per_query, const float att_cap,
MatPtrT<float>& att_out, float* HWY_RESTRICT exp_denominator_sums,
float* HWY_RESTRICT max_logits) {
float* HWY_RESTRICT max_logits,
hwy::AlignedVector<uint8_t>* worker_workspace) {
TileFlashAttentionReturnExpSumsAndMaxLogitsAMX(
kvs, q_count, q_base, start_pos_per_query, last_pos_per_query, att_cap,
att_out, exp_denominator_sums, max_logits);
att_out, exp_denominator_sums, max_logits, worker_workspace);
}

void DispatchTileFlashAttentionReturnExpSumsAndMaxLogitsAMXInt8(
Expand Down
3 changes: 2 additions & 1 deletion gemma/flash_attention.h
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,8 @@ namespace gcpp {
hwy::Span<const size_t> start_pos_per_query, \
hwy::Span<const size_t> last_pos_per_query, const float att_cap, \
MatPtrT<float>& att_out, float* HWY_RESTRICT exp_denominator_sums, \
float* HWY_RESTRICT max_logits); \
float* HWY_RESTRICT max_logits, \
hwy::AlignedVector<uint8_t>* worker_workspace); \
\
void DispatchTileFlashAttentionReturnExpSumsAndMaxLogitsAMXInt8( \
hwy::Span<const MatPtr> kvs, size_t q_count, \
Expand Down
6 changes: 3 additions & 3 deletions gemma/flash_attention_amx-inl.h
Original file line number Diff line number Diff line change
Expand Up @@ -1392,7 +1392,8 @@ inline void TileFlashAttentionReturnExpSumsAndMaxLogitsAMX(
hwy::Span<const size_t> start_pos_per_query,
hwy::Span<const size_t> last_pos_per_query, const float att_cap,
MatPtrT<float>& att_out, float* HWY_RESTRICT exp_denominator_sums,
float* HWY_RESTRICT max_logits) {
float* HWY_RESTRICT max_logits,
hwy::AlignedVector<uint8_t>* worker_workspace = nullptr) {
#if GEMMA_HAVE_AMX
if (hwy::HaveTile64BMatMulBF16()) {
TileFlashAttentionAMX_BF16_Impl(kvs, q_count, q_base, start_pos_per_query,
Expand All @@ -1403,8 +1404,7 @@ inline void TileFlashAttentionReturnExpSumsAndMaxLogitsAMX(
#endif // GEMMA_HAVE_AMX
DispatchTileFlashAttentionReturnExpSumsAndMaxLogitsBF16(
kvs, q_count, q_base, start_pos_per_query, last_pos_per_query, att_cap,
att_out, exp_denominator_sums, max_logits,
/*worker_workspace=*/nullptr);
att_out, exp_denominator_sums, max_logits, worker_workspace);
}

inline void TileFlashAttentionReturnExpSumsAndMaxLogitsAMXInt8(
Expand Down
8 changes: 5 additions & 3 deletions gemma/flash_attention_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -700,7 +700,8 @@ void RunTiledFlashAttentionTest(gcpp::KVEncoding kv_encoding,
kvs, num_queries, bf16_queries.data(),
hwy::Span<const size_t>(start_pos_per_query),
hwy::Span<const size_t>(last_pos_per_query), att_cap, att_out,
exp_denominator_sums.data(), max_logits.data());
exp_denominator_sums.data(), max_logits.data(),
/*worker_workspace=*/nullptr);
} else if (attention_impl == AttentionImpl::kFlashAMXInt8) {
std::vector<int8_t, hwy::AlignedAllocator<int8_t>> int8_queries(
num_queries * qkv_dim);
Expand Down Expand Up @@ -998,7 +999,7 @@ struct TiledFlashAttentionTestSetup {
case TiledTestKernel::kAmxBF16:
DispatchTileFlashAttentionReturnExpSumsAndMaxLogitsAMX(
kvs, q_count, q_bf16.data() + first_q * kQkvDim, starts, lasts,
/*att_cap=*/0.0f, att_out, exp_sums, max_logits);
/*att_cap=*/0.0f, att_out, exp_sums, max_logits, workspace);
break;
case TiledTestKernel::kAmxInt8:
DispatchTileFlashAttentionReturnExpSumsAndMaxLogitsAMXInt8(
Expand Down Expand Up @@ -1453,7 +1454,8 @@ void RunTiledFlashAttentionDifferentialTest(
kvs, num_queries, bf16_queries.data(),
hwy::Span<const size_t>(start_pos_per_query),
hwy::Span<const size_t>(last_pos_per_query), att_cap, att_out,
exp_denominator_sums.data(), max_logits.data());
exp_denominator_sums.data(), max_logits.data(),
/*worker_workspace=*/nullptr);
} else if (opt_impl == AttentionImpl::kFlashAMXInt8) {
DispatchTileFlashAttentionReturnExpSumsAndMaxLogitsAMXInt8(
kvs, num_queries, int8_queries.data(), q_scales,
Expand Down
112 changes: 88 additions & 24 deletions gemma/kv_cache.cc
Original file line number Diff line number Diff line change
Expand Up @@ -280,34 +280,46 @@ KVCache::KVCache(const ModelConfig& config, const InferenceArgs& inference_args,

if (runtime_config.attention_impl == AttentionImpl::kFlash ||
IsTiledAttention(runtime_config.attention_impl)) {
const bool tiled_only = IsTiledAttention(runtime_config.attention_impl) &&
!config.is_encoder_decoder && !config.HasMLA() &&
config.num_mtp_layers == 0;
kv_is_scratch = runtime_config.attention_impl == AttentionImpl::kFlash &&
!config.is_encoder_decoder && !config.HasMLA();
size_t projection_cols = config.KVCacheCols();
size_t projection_rows = CappedSeqLen(config, inference_args);
if (kv_is_scratch) {
projection_cols = 0;
for (const LayerConfig& layer : kv_layer_configs) {
projection_cols = std::max(projection_cols, layer.CacheLayerSize());
if (tiled_only) {
// TiledAttention exclusively reads and writes compact_local_kv_cache_ptr
// and compact_global_kv_cache_ptr. Keep a 1x1 sentinel in kv_cache so
// KVCachePtr::IsEmpty() returns false without allocating O(seq_len)
// unused flat/transposed buffers (36 GiB saved at 1M context).
kv_cache = MatStorageT<KV_t>("kv", Extents2D(1, 1), allocator,
MatPadding::kOdd);
} else {
size_t projection_cols = config.KVCacheCols();
size_t projection_rows = CappedSeqLen(config, inference_args);
if (kv_is_scratch) {
projection_cols = 0;
for (const LayerConfig& layer : kv_layer_configs) {
projection_cols = std::max(projection_cols, layer.CacheLayerSize());
}
projection_rows =
std::min(projection_rows,
std::max(size_t{1}, runtime_config.prefill_tbatch_size));
}
projection_rows =
std::min(projection_rows,
std::max(size_t{1}, runtime_config.prefill_tbatch_size));
kv_cache =
MatStorageT<KV_t>("kv", Extents2D(projection_rows, projection_cols),
allocator, MatPadding::kOdd);
k_cache = MatStorageT<KV_t>(
"k",
Extents2D(hwy::RoundUpTo(CappedSeqLen(config, inference_args),
kMaxBF16PerVector),
k_v_cols),
allocator, MatPadding::kPacked);
v_cache = MatStorageT<KV_t>(
"v",
Extents2D(hwy::RoundUpTo(CappedSeqLen(config, inference_args),
kMaxBF16PerVector),
k_v_cols),
allocator, MatPadding::kPacked);
}
kv_cache =
MatStorageT<KV_t>("kv", Extents2D(projection_rows, projection_cols),
allocator, MatPadding::kOdd);
k_cache = MatStorageT<KV_t>(
"k",
Extents2D(hwy::RoundUpTo(CappedSeqLen(config, inference_args),
kMaxBF16PerVector),
k_v_cols),
allocator, MatPadding::kPacked);
v_cache = MatStorageT<KV_t>(
"v",
Extents2D(hwy::RoundUpTo(CappedSeqLen(config, inference_args),
kMaxBF16PerVector),
k_v_cols),
allocator, MatPadding::kPacked);
const size_t num_tiles =
hwy::DivCeil(CappedSeqLen(config, inference_args), kTileSize);
tiled_seq_len = num_tiles * kTileSize;
Expand Down Expand Up @@ -489,6 +501,58 @@ void KVCache::ZeroInit() {
if (ds_state_snapshot.HasPtr()) gcpp::ZeroInit(ds_state_snapshot);
}

void KVCache::ZeroInitPrefix(size_t dirty_tokens) {
if (dirty_tokens == 0) return;

if (!kv_head_ptrs.empty()) {
const size_t needed_tiles = hwy::DivCeil(dirty_tokens, kTileSize) + 1;
for (MatPtr& hp : kv_head_ptrs) {
if (!hp.HasPtr()) continue;
const size_t dirty_tiles = std::min(hp.Rows(), needed_tiles);
hwy::ZeroBytes(hp.RowBytes(0),
dirty_tiles * hp.Stride() * hp.ElementBytes());
}
} else {
if (compact_local_kv_cache_ptr.HasPtr()) {
gcpp::ZeroInit(compact_local_kv_cache_ptr);
}
if (compact_global_kv_cache_ptr.HasPtr()) {
gcpp::ZeroInit(compact_global_kv_cache_ptr);
}
}

if (kv_cache.HasPtr()) {
if (kv_is_scratch || kv_cache.Rows() <= dirty_tokens) {
gcpp::ZeroInit(kv_cache);
} else {
hwy::ZeroBytes(kv_cache.RowBytes(0),
dirty_tokens * kv_cache.Stride() *
kv_cache.ElementBytes());
}
}

auto zero_kv_prefix = [&](MatStorageT<KV_t>& cache) {
if (!cache.HasPtr()) return;
size_t dirty_rows = cache.Rows();
if (k_v_cols > 0 && cache.Cols() > k_v_cols) {
const size_t group = cache.Cols() / k_v_cols;
dirty_rows =
std::min(cache.Rows(), hwy::DivCeil(dirty_tokens, group) + 1);
} else {
dirty_rows = std::min(
cache.Rows(),
hwy::RoundUpTo(dirty_tokens, kMaxBF16PerVector) + kMaxBF16PerVector);
}
hwy::ZeroBytes(cache.RowBytes(0),
dirty_rows * cache.Stride() * cache.ElementBytes());
};
zero_kv_prefix(k_cache);
zero_kv_prefix(v_cache);

if (ds_state.HasPtr()) gcpp::ZeroInit(ds_state);
if (ds_state_snapshot.HasPtr()) gcpp::ZeroInit(ds_state_snapshot);
}

void KVCachePtr::ZeroInit() {
if (cache != nullptr) {
cache->ZeroInit();
Expand Down
3 changes: 3 additions & 0 deletions gemma/kv_cache.h
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ struct KVCache {
// copy ctor to make the cost explicit.
KVCache Copy();
void ZeroInit();
// Zeroes only KV cache storage that could have been dirtied by evaluating up
// to `dirty_tokens` positions starting from position 0.
void ZeroInitPrefix(size_t dirty_tokens);

// Default flash only needs the current layer's projections until transpose.
// Other backends can still use kv_cache as persistent inference state.
Expand Down
2 changes: 1 addition & 1 deletion gemma/tiled_attention.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1480,7 +1480,7 @@ void LocalAttentionForAllHeadsTokensAndBatch(
hwy::Span<const size_t>(start_pos_per_query),
hwy::Span<const size_t>(last_pos_per_query),
activations.config.att_cap, att_out, exp_denominator_sums.data(),
max_logits.data());
max_logits.data(), worker_workspace);

} else if (attention_impl == AttentionImpl::kFlashTransposedQsInt16) {
HWY_DASSERT(activations.int16_queries != nullptr);
Expand Down
60 changes: 53 additions & 7 deletions python/gemma_py.cc
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,11 @@
#include <pybind11/stl.h>

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <limits>
#include <memory>
#include <optional>
#include <set>
#include <stdexcept>
#include <string>
Expand All @@ -47,7 +51,11 @@ static void RemoveTrailingZeros(std::vector<int> &vec) {
// Wrapper around GemmaEnv to expose to Python.
class GemmaModel {
public:
GemmaModel(const gcpp::GemmaArgs& args) : env_(args), last_prob_(0.0f) {}
GemmaModel(const gcpp::GemmaArgs& args, size_t max_seq_len = 0)
: env_(args),
last_prob_(0.0f),
max_seq_len_(max_seq_len > 0 ? max_seq_len
: env_.MutableKVCache().SeqLen()) {}

// Generates a single example, given a prompt and a callback to stream the
// generated tokens.
Expand Down Expand Up @@ -282,39 +290,76 @@ class GemmaModel {

float GetLastProb() const { return last_prob_; }

std::vector<int> Tokenize(const std::string& text) const {
return env_.Tokenize(text);
}

std::string Detokenize(const std::vector<int>& token_ids) const {
return env_.StringFromTokens(token_ids);
}

bool ModelIsLoaded() const { return env_.GetGemma() != nullptr; }

private:

gcpp::GemmaEnv env_;
std::unique_ptr<gcpp::ImageTokens> image_tokens_;
std::optional<gcpp::KVCache> eval_kv_cache_;
std::unique_ptr<gcpp::Activations> eval_activations_;
size_t eval_activations_max_tbatch_ = 0;
size_t eval_activations_kv_seq_len_ = 0;
gcpp::AttentionImpl eval_activations_att_impl_ = gcpp::AttentionImpl::kFlash;
size_t dirty_kv_tokens_ = std::numeric_limits<size_t>::max();
gcpp::TimingInfo timing_info_;
float last_prob_;
size_t max_seq_len_ = 0;
};

PYBIND11_MODULE(gemma, mod) {
py::class_<GemmaModel>(mod, "GemmaModel")
py::class_<GemmaModel> gemma_cls(mod, "GemmaModel");
gemma_cls
.def(py::init([](const std::string& tokenizer, const std::string& weights,
size_t max_threads) {
size_t max_threads, size_t seq_len,
size_t prefill_tbatch_size,
const std::string& attention_impl, bool map_weights) {
gcpp::ThreadingArgs threading;
threading.max_lps = max_threads;

gcpp::InferenceArgs inference;
inference.max_generated_tokens = 512;
if (seq_len > 0) {
// Reserve one extra SIMD row tile (kMaxBF16PerVector) in the
// initial KVCache so AVX-512 batched prefill does not need to
// allocate a second KVCache in ComputeLikelihood.
inference.seq_len =
hwy::RoundUpTo(seq_len, gcpp::kMaxBF16PerVector) +
gcpp::kMaxBF16PerVector;
}
if (prefill_tbatch_size > 0) {
inference.prefill_tbatch_size = prefill_tbatch_size;
}
if (attention_impl == "auto") {
inference.attention_impl = "flash_amx";
} else if (!attention_impl.empty()) {
inference.attention_impl = attention_impl;
}

const gcpp::GemmaArgs args(gcpp::LoaderArgs(tokenizer, weights),
threading, inference);
auto gemma = std::make_unique<GemmaModel>(args);
gcpp::LoaderArgs loader_args(tokenizer, weights);
if (map_weights) {
loader_args.map = gcpp::Tristate::kTrue;
}
const gcpp::GemmaArgs args(loader_args, threading, inference);
auto gemma = std::make_unique<GemmaModel>(args, seq_len);
if (!gemma->ModelIsLoaded()) {
throw std::invalid_argument("Could not load model.");
}
return gemma;
}),
py::arg("tokenizer_path"), py::arg("weights_path"),
py::arg("max_threads") = 0)
py::arg("max_threads") = 0, py::arg("seq_len") = 0,
py::arg("prefill_tbatch_size") = 0,
py::arg("attention_impl") = "flash",
py::arg("map_weights") = false)
// seed arguments are ignored.
.def("generate_ex", &GemmaModel::GenerateEx, py::arg("prompt"),
py::arg("stream"), py::arg("max_generated_tokens") = 1024,
Expand All @@ -338,5 +383,6 @@ PYBIND11_MODULE(gemma, mod) {
.def("score_next", &GemmaModel::ScoreNext, py::arg("prompt"),
py::arg("candidates"))
.def("get_last_prob", &GemmaModel::GetLastProb)
.def("tokenize", &GemmaModel::Tokenize, py::arg("text"))
.def("detokenize", &GemmaModel::Detokenize, py::arg("token_ids"));
}
Loading