From 0ca6ed80a89649177b775d68e4d3d959078480f2 Mon Sep 17 00:00:00 2001 From: Dani Ferreira Franco Moura Date: Tue, 15 Sep 2026 13:30:08 -0700 Subject: [PATCH] Move MergeOnlineSoftmax into a new per-target header, so it can be reused across attention translation units. No functional change. PiperOrigin-RevId: 982008282 --- BUILD.bazel | 1 + gemma/flash_attention.cc | 1 + gemma/online_softmax-inl.h | 81 ++++++++++++++++++++++++++++++++++++++ gemma/tiled_attention.cc | 29 +------------- 4 files changed, 84 insertions(+), 28 deletions(-) create mode 100644 gemma/online_softmax-inl.h diff --git a/BUILD.bazel b/BUILD.bazel index 4f58cfe1..93765721 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -839,6 +839,7 @@ cc_library( textual_hdrs = [ "gemma/gemma-inl.h", "gemma/flash_attention_arm-inl.h", + "gemma/online_softmax-inl.h", ], deps = [ ":activations", diff --git a/gemma/flash_attention.cc b/gemma/flash_attention.cc index a8e7cda1..5c0d2b7a 100644 --- a/gemma/flash_attention.cc +++ b/gemma/flash_attention.cc @@ -55,6 +55,7 @@ #include "gemma/attention.h" #include "gemma/flash_attention.h" #include "gemma/flash_attention_arm-inl.h" +#include "gemma/online_softmax-inl.h" #include "ops/matmul-inl.h" #include "ops/ops-inl.h" #include "hwy/contrib/math/fast_math-inl.h" diff --git a/gemma/online_softmax-inl.h b/gemma/online_softmax-inl.h new file mode 100644 index 00000000..4b5cc984 --- /dev/null +++ b/gemma/online_softmax-inl.h @@ -0,0 +1,81 @@ +// Copyright 2025 Google LLC +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Online softmax state merge kernel shared across attention implementations. + +#include +#include + +#include +#include + +// Include guard (still compiled once per target) +#if defined(THIRD_PARTY_GEMMA_CPP_GEMMA_ONLINE_SOFTMAX_INL_H_) == \ + defined(HWY_TARGET_TOGGLE) +#ifdef THIRD_PARTY_GEMMA_CPP_GEMMA_ONLINE_SOFTMAX_INL_H_ +#undef THIRD_PARTY_GEMMA_CPP_GEMMA_ONLINE_SOFTMAX_INL_H_ +#else +#define THIRD_PARTY_GEMMA_CPP_GEMMA_ONLINE_SOFTMAX_INL_H_ +#endif + +#include "hwy/highway.h" +// After highway.h +#include "ops/ops-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace gcpp { +namespace HWY_NAMESPACE { + +// Folds the online-softmax state `other` into `accumulator` in place, so that +// the result is as if both halves had been softmaxed together: the two +// attention outputs are rescaled by their renormalized denominators and +// summed. `*_att_out` hold `qkv_dim` floats each, `*_softmax_max` the running +// max logit and `*_softmax_d` the running sum of exponentials. A zero +// denominator marks an empty state, which is the identity of this operation. +HWY_INLINE void MergeOnlineSoftmax(const float* HWY_RESTRICT other_att_out, + const float other_softmax_max, + const float other_softmax_d, size_t qkv_dim, + float* HWY_RESTRICT accumulator_att_out, + float& accumulator_softmax_max, + float& accumulator_softmax_d) { + if (other_softmax_d == 0.0f) { + return; + } + if (accumulator_softmax_d == 0.0f) { + memcpy(accumulator_att_out, other_att_out, + qkv_dim * sizeof(*accumulator_att_out)); + accumulator_softmax_max = other_softmax_max; + accumulator_softmax_d = other_softmax_d; + return; + } + const float m_new = std::max(accumulator_softmax_max, other_softmax_max); + const float exp_l = std::exp(accumulator_softmax_max - m_new); + const float exp_r = std::exp(other_softmax_max - m_new); + const float d_new = accumulator_softmax_d * exp_l + other_softmax_d * exp_r; + const float d_new_inv = 1.0f / d_new; + const float c1 = accumulator_softmax_d * exp_l * d_new_inv; + const float c2 = other_softmax_d * exp_r * d_new_inv; + MulByConst(c1, accumulator_att_out, qkv_dim); + MulByConstAndAdd(c2, other_att_out, accumulator_att_out, qkv_dim); + accumulator_softmax_max = m_new; + accumulator_softmax_d = d_new; +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace gcpp +HWY_AFTER_NAMESPACE(); + +#endif // THIRD_PARTY_GEMMA_CPP_GEMMA_ONLINE_SOFTMAX_INL_H_ diff --git a/gemma/tiled_attention.cc b/gemma/tiled_attention.cc index 14173c1d..ebd6d6b1 100644 --- a/gemma/tiled_attention.cc +++ b/gemma/tiled_attention.cc @@ -39,40 +39,13 @@ #include "gemma/attention.h" #include "gemma/flash_attention.h" // includes highway.h #include "gemma/gemma-inl.h" +#include "gemma/online_softmax-inl.h" #include "ops/ops-inl.h" HWY_BEFORE_NAMESPACE(); namespace gcpp { namespace HWY_NAMESPACE { -static HWY_INLINE void MergeOnlineSoftmax( - const float* HWY_RESTRICT other_att_out, const float other_softmax_max, - const float other_softmax_d, size_t qkv_dim, - float* HWY_RESTRICT accumulator_att_out, float& accumulator_softmax_max, - float& accumulator_softmax_d) { - if (other_softmax_d == 0.0f) { - return; - } - if (accumulator_softmax_d == 0.0f) { - memcpy(accumulator_att_out, other_att_out, - qkv_dim * sizeof(*accumulator_att_out)); - accumulator_softmax_max = other_softmax_max; - accumulator_softmax_d = other_softmax_d; - return; - } - const float m_new = std::max(accumulator_softmax_max, other_softmax_max); - const float exp_l = std::exp(accumulator_softmax_max - m_new); - const float exp_r = std::exp(other_softmax_max - m_new); - const float d_new = accumulator_softmax_d * exp_l + other_softmax_d * exp_r; - const float d_new_inv = 1.0f / d_new; - const float c1 = accumulator_softmax_d * exp_l * d_new_inv; - const float c2 = other_softmax_d * exp_r * d_new_inv; - MulByConst(c1, accumulator_att_out, qkv_dim); - MulByConstAndAdd(c2, other_att_out, accumulator_att_out, qkv_dim); - accumulator_softmax_max = m_new; - accumulator_softmax_d = d_new; -} - static constexpr size_t kMergeGroupSize = 32; static HWY_INLINE void MergeOnlineSoftmaxGroup32(