From 1d3496bb14df3f868b57c06bb54cea8df9085f73 Mon Sep 17 00:00:00 2001 From: sohailshk <122166523+sohailshk@users.noreply.github.com> Date: Fri, 13 Nov 2026 01:50:38 +0530 Subject: [PATCH] Fix TopK returning unaccepted entries when accept_token allows fewer than k TopK sorted and unpacked k entries from a vector that only holds the tokens accepted by accept_token. When fewer than k tokens were accepted (for example RuntimeConfig::top_k = 40 with an accept_token that allows a handful of tokens; reachable through the C++ API, hello_world, run_mmlu or the Python accept= argument), it called VQSelect/VQSort with k greater than the number of elements and returned k entries, the surplus being uninitialized memory from the reserved capacity. FusedSoftmaxAndSampleTopK, which already limits itself to token_logits.size(), then included those entries in the softmax, so a token that accept_token had rejected (in practice token 0) could be sampled. Clamp to the number of accepted tokens and return fewer than k entries in that case. Skip VQSelect when every accepted token is kept, because Highway's Select requires k < num. TopK asserts that at least one token was accepted, and SampleTopK uses the returned size. --- ops/ops-inl.h | 24 ++++++++++++++++-------- ops/ops_test.cc | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/ops/ops-inl.h b/ops/ops-inl.h index 97c9d0a9..963c05f2 100644 --- a/ops/ops-inl.h +++ b/ops/ops-inl.h @@ -1602,13 +1602,19 @@ HWY_NOINLINE HWY_MAYBE_UNUSED std::vector TopK( } } - hwy::VQSelect(packed_token_probs.data(), packed_token_probs.size(), k, - hwy::SortDescending()); - hwy::VQSort(packed_token_probs.data(), k, hwy::SortDescending()); + // `accept_token` may have rejected so many tokens that fewer than `k` remain. + HWY_ASSERT(!packed_token_probs.empty()); // at least one token must be OK + const size_t num = HWY_MIN(k, packed_token_probs.size()); + // `VQSelect` requires `num < size`; nothing to select otherwise. + if (num < packed_token_probs.size()) { + hwy::VQSelect(packed_token_probs.data(), packed_token_probs.size(), num, + hwy::SortDescending()); + } + hwy::VQSort(packed_token_probs.data(), num, hwy::SortDescending()); std::vector token_probs; - token_probs.reserve(k); - for (int32_t i = 0; i < static_cast(k); ++i) { + token_probs.reserve(num); + for (size_t i = 0; i < num; ++i) { token_probs.push_back(UnpackTokenAndProb(packed_token_probs[i])); } return token_probs; @@ -1619,9 +1625,11 @@ HWY_NOINLINE HWY_MAYBE_UNUSED int SampleTopK(Logits logits, size_t k, RngStream& gen, float temperature, TAcceptToken& accept_token) { std::vector token_probs = TopK(logits, k, accept_token); - std::vector topk_indices(k); - std::vector topk_probs(k); - for (size_t i = 0; i < k; ++i) { + // May be shorter than `k` if `accept_token` rejected most tokens. + const size_t num = token_probs.size(); + std::vector topk_indices(num); + std::vector topk_probs(num); + for (size_t i = 0; i < num; ++i) { topk_indices[i] = token_probs[i].token; topk_probs[i] = token_probs[i].prob; } diff --git a/ops/ops_test.cc b/ops/ops_test.cc index 651d4540..923b9e48 100644 --- a/ops/ops_test.cc +++ b/ops/ops_test.cc @@ -914,6 +914,43 @@ void TestTopK() { } } +// If `accept_token` leaves fewer than `k` candidates, `TopK` and `SampleTopK` +// must only consider the accepted tokens. +void TestTopKFewerAcceptedThanK() { + const size_t kSize = 300; + std::vector logits_vec(kSize); + // Negative logits, so that any extra zero-valued entry would outrank the + // eligible tokens. + for (size_t i = 0; i < kSize; ++i) logits_vec[i] = -10.0f - 0.01f * i; + Logits logits(logits_vec.data(), kSize); + + // Only tokens 10, 20 and 30 are eligible, but k = 8. + std::function accept_token = [](int i, float) { + return i == 10 || i == 20 || i == 30; + }; + const std::vector top = TopK(logits, /*k=*/8, accept_token); + ASSERT_EQ(top.size(), size_t{3}); + EXPECT_EQ(top[0].token, 10); + EXPECT_EQ(top[1].token, 20); + EXPECT_EQ(top[2].token, 30); + + RngStream rng = MakeRng(); + for (float temperature : {0.0f, 1.0f}) { + for (int i = 0; i < 100; ++i) { + const int sample = + SampleTopK(logits, /*k=*/8, rng, temperature, accept_token); + EXPECT_TRUE(sample == 10 || sample == 20 || sample == 30) + << "sample " << sample << " temperature " << temperature; + // The fused softmax + sampling path used by `gemma.cc`. + const TokenAndProb fused = FusedSoftmaxAndSampleTopK( + logits, /*k=*/8, rng, temperature, accept_token, Ctx(), /*worker=*/0); + EXPECT_TRUE(fused.token == 10 || fused.token == 20 || fused.token == 30) + << "fused " << fused.token << " temperature " << temperature; + EXPECT_GT(fused.prob, 0.0f); + } + } +} + void TestPackTokenAndProb() { double packed1 = PackTokenAndProb(10, 0.96f); TokenAndProb unpacked1 = UnpackTokenAndProb(packed1); @@ -953,6 +990,7 @@ HWY_EXPORT_AND_TEST_P(OpsTest, TestAllLayerNorm); HWY_EXPORT_AND_TEST_P(OpsTest, TestLayerNormSimple); HWY_EXPORT_AND_TEST_P(OpsTest, TestSampleTopK); HWY_EXPORT_AND_TEST_P(OpsTest, TestTopK); +HWY_EXPORT_AND_TEST_P(OpsTest, TestTopKFewerAcceptedThanK); HWY_EXPORT_AND_TEST_P(OpsTest, TestPackTokenAndProb); HWY_AFTER_TEST();