Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 16 additions & 8 deletions ops/ops-inl.h
Original file line number Diff line number Diff line change
Expand Up @@ -1602,13 +1602,19 @@ HWY_NOINLINE HWY_MAYBE_UNUSED std::vector<TokenAndProb> TopK(
}
}

hwy::VQSelect(packed_token_probs.data(), packed_token_probs.size(), k,
hwy::SortDescending());
hwy::VQSort(packed_token_probs.data(), k, hwy::SortDescending());
// `accept_token` may have rejected so many tokens that fewer than `k` remain.
HWY_ASSERT(!packed_token_probs.empty()); // at least one token must be OK
const size_t num = HWY_MIN(k, packed_token_probs.size());
// `VQSelect` requires `num < size`; nothing to select otherwise.
if (num < packed_token_probs.size()) {
hwy::VQSelect(packed_token_probs.data(), packed_token_probs.size(), num,
hwy::SortDescending());
}
hwy::VQSort(packed_token_probs.data(), num, hwy::SortDescending());

std::vector<TokenAndProb> token_probs;
token_probs.reserve(k);
for (int32_t i = 0; i < static_cast<int32_t>(k); ++i) {
token_probs.reserve(num);
for (size_t i = 0; i < num; ++i) {
token_probs.push_back(UnpackTokenAndProb(packed_token_probs[i]));
}
return token_probs;
Expand All @@ -1619,9 +1625,11 @@ HWY_NOINLINE HWY_MAYBE_UNUSED int SampleTopK(Logits logits, size_t k,
RngStream& gen, float temperature,
TAcceptToken& accept_token) {
std::vector<TokenAndProb> token_probs = TopK(logits, k, accept_token);
std::vector<int> topk_indices(k);
std::vector<float> topk_probs(k);
for (size_t i = 0; i < k; ++i) {
// May be shorter than `k` if `accept_token` rejected most tokens.
const size_t num = token_probs.size();
std::vector<int> topk_indices(num);
std::vector<float> topk_probs(num);
for (size_t i = 0; i < num; ++i) {
topk_indices[i] = token_probs[i].token;
topk_probs[i] = token_probs[i].prob;
}
Expand Down
38 changes: 38 additions & 0 deletions ops/ops_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -914,6 +914,43 @@ void TestTopK() {
}
}

// If `accept_token` leaves fewer than `k` candidates, `TopK` and `SampleTopK`
// must only consider the accepted tokens.
void TestTopKFewerAcceptedThanK() {
const size_t kSize = 300;
std::vector<float> logits_vec(kSize);
// Negative logits, so that any extra zero-valued entry would outrank the
// eligible tokens.
for (size_t i = 0; i < kSize; ++i) logits_vec[i] = -10.0f - 0.01f * i;
Logits logits(logits_vec.data(), kSize);

// Only tokens 10, 20 and 30 are eligible, but k = 8.
std::function<bool(int, float)> accept_token = [](int i, float) {
return i == 10 || i == 20 || i == 30;
};
const std::vector<TokenAndProb> top = TopK(logits, /*k=*/8, accept_token);
ASSERT_EQ(top.size(), size_t{3});
EXPECT_EQ(top[0].token, 10);
EXPECT_EQ(top[1].token, 20);
EXPECT_EQ(top[2].token, 30);

RngStream rng = MakeRng();
for (float temperature : {0.0f, 1.0f}) {
for (int i = 0; i < 100; ++i) {
const int sample =
SampleTopK(logits, /*k=*/8, rng, temperature, accept_token);
EXPECT_TRUE(sample == 10 || sample == 20 || sample == 30)
<< "sample " << sample << " temperature " << temperature;
// The fused softmax + sampling path used by `gemma.cc`.
const TokenAndProb fused = FusedSoftmaxAndSampleTopK(
logits, /*k=*/8, rng, temperature, accept_token, Ctx(), /*worker=*/0);
EXPECT_TRUE(fused.token == 10 || fused.token == 20 || fused.token == 30)
<< "fused " << fused.token << " temperature " << temperature;
EXPECT_GT(fused.prob, 0.0f);
}
}
}

void TestPackTokenAndProb() {
double packed1 = PackTokenAndProb(10, 0.96f);
TokenAndProb unpacked1 = UnpackTokenAndProb(packed1);
Expand Down Expand Up @@ -953,6 +990,7 @@ HWY_EXPORT_AND_TEST_P(OpsTest, TestAllLayerNorm);
HWY_EXPORT_AND_TEST_P(OpsTest, TestLayerNormSimple);
HWY_EXPORT_AND_TEST_P(OpsTest, TestSampleTopK);
HWY_EXPORT_AND_TEST_P(OpsTest, TestTopK);
HWY_EXPORT_AND_TEST_P(OpsTest, TestTopKFewerAcceptedThanK);
HWY_EXPORT_AND_TEST_P(OpsTest, TestPackTokenAndProb);
HWY_AFTER_TEST();

Expand Down