Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
1c16b5f
returned session id and used it in evals.
ShaharKatz Aug 26, 2025
795a187
added catching an import error on VertexAiSessionService
ShaharKatz Aug 30, 2025
f41d17e
erased the todo in trying to import vertex after fixing the todo
ShaharKatz Aug 30, 2025
df5cb0a
the session_id is used if it not null or the vertex session service i…
ShaharKatz Aug 30, 2025
7d1c90f
fixing the following the comment in the PR - The create_session meth…
ShaharKatz Aug 30, 2025
2dacd2a
added added BaseSessionIdSupplier and PreCreatedSessionIdSupplier
ShaharKatz Sep 6, 2025
c031ec6
reverted VertexAiSessionService specific logic, upgraded session_id_s…
ShaharKatz Sep 6, 2025
6556ba1
added SessionIdSupplier logic to local eval service
ShaharKatz Sep 6, 2025
286de8d
fixed bug in session id assignment and removed session id from being …
ShaharKatz Sep 6, 2025
a23ceb4
removed reduncant created session id
ShaharKatz Sep 7, 2025
f800dbc
added a test_user_id in case it's not given instead of none and added…
ShaharKatz Sep 7, 2025
4110a13
upgraded test to return the correct session structure returned by the…
ShaharKatz Sep 20, 2025
e539232
added BaseSessionIdSupplier to the LocalEvalService
ShaharKatz Sep 20, 2025
108c58b
upgraded documentation to conform to the session interface instead of…
ShaharKatz Sep 20, 2025
b1d0fd9
removed the old session as dict
ShaharKatz Sep 20, 2025
75d3b63
reverted _generate_inferences_from_root_agento again only return invo…
ShaharKatz Sep 20, 2025
133a9e8
removed TODOs that were fixed and cleaned commented out code
ShaharKatz Sep 20, 2025
080ebef
changed single quotes to double quotes
ShaharKatz Sep 20, 2025
df9f758
reverted the session_id return as part of _process_query
ShaharKatz Sep 20, 2025
2f575ae
reverted the changes to llms-full.txt
ShaharKatz Sep 20, 2025
3991859
aligned with main
ShaharKatz Nov 20, 2025
97472c7
Merge branch 'main' into feature/Running-Eval-doesnt-work-with-Vertex…
ShaharKatz Nov 20, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 37 additions & 4 deletions src/google/adk/evaluation/evaluation_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@
from ..runners import Runner
from ..sessions.base_session_service import BaseSessionService
from ..sessions.in_memory_session_service import InMemorySessionService
from ..sessions.base_session_id_supplier import BaseSessionIdSupplier
from ..sessions.pre_created_session_id_supplier import PreCreatedSessionIdSupplier
from ..sessions.session import Session
from ..utils.context_utils import Aclosing
from ._retry_options_utils import EnsureRetryOptionsPlugin
Expand Down Expand Up @@ -192,6 +194,7 @@ async def _generate_inferences_from_root_agent(
initial_session: Optional[SessionInput] = None,
session_id: Optional[str] = None,
session_service: Optional[BaseSessionService] = None,
session_id_supplier: Optional[BaseSessionIdSupplier] = None,
artifact_service: Optional[BaseArtifactService] = None,
memory_service: Optional[BaseMemoryService] = None,
) -> list[Invocation]:
Expand All @@ -207,14 +210,44 @@ async def _generate_inferences_from_root_agent(
initial_session.app_name if initial_session else "EvaluationGenerator"
)
user_id = initial_session.user_id if initial_session else "test_user_id"
session_id = session_id if session_id else str(uuid.uuid4())

_ = await session_service.create_session(
# if session_is is provided, check if the session exists
if session_id:
session = await session_service.get_session(
app_name=app_name,
user_id=user_id,
state=initial_session.state if initial_session else {},
session_id=session_id,
)
)
if session:
# session exists, do nothing
pass
else:
# session does not exist, create a new session
session = await session_service.create_session(
app_name=app_name,
user_id=user_id,
state=initial_session.state if initial_session else {},
session_id=session_id,
)

session_id = session.id # TODO: some other service didn't have this..
else:
# session_id is not provided, create a new session id from the provider
if not session_id_supplier:
session_id_supplier_service = PreCreatedSessionIdSupplier(session_service)

session_id = await session_id_supplier_service.get_session_id(
app_name=app_name,
user_id=user_id,
initial_state=initial_session.state if initial_session else {}
)

_ = await session_service.create_session(
app_name=app_name,
user_id=user_id,
state=initial_session.state if initial_session else {},
session_id=session_id
)

if not artifact_service:
artifact_service = InMemoryArtifactService()
Expand Down
19 changes: 12 additions & 7 deletions src/google/adk/evaluation/local_eval_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
from ..memory.base_memory_service import BaseMemoryService
from ..sessions.base_session_service import BaseSessionService
from ..sessions.in_memory_session_service import InMemorySessionService
from ..sessions.base_session_id_supplier import BaseSessionIdSupplier
from ..sessions.pre_created_session_id_supplier import PreCreatedSessionIdSupplier
from ..utils.feature_decorator import experimental
from .base_eval_service import BaseEvalService
from .base_eval_service import EvaluateConfig
Expand Down Expand Up @@ -60,10 +62,6 @@
EVAL_SESSION_ID_PREFIX = '___eval___session___'


def _get_session_id() -> str:
return f'{EVAL_SESSION_ID_PREFIX}{str(uuid.uuid4())}'


@experimental
class LocalEvalService(BaseEvalService):
"""An implementation of BaseEvalService, that runs the evals locally."""
Expand All @@ -74,10 +72,10 @@ def __init__(
eval_sets_manager: EvalSetsManager,
metric_evaluator_registry: Optional[MetricEvaluatorRegistry] = None,
session_service: Optional[BaseSessionService] = None,
session_id_supplier: BaseSessionIdSupplier = None,
artifact_service: Optional[BaseArtifactService] = None,
eval_set_results_manager: Optional[EvalSetResultsManager] = None,
session_id_supplier: Callable[[], str] = _get_session_id,
user_simulator_provider: UserSimulatorProvider = UserSimulatorProvider(),
user_simulator_provider: Optional[UserSimulatorProvider] = UserSimulatorProvider(),
memory_service: Optional[BaseMemoryService] = None,
):
self._root_agent = root_agent
Expand All @@ -87,6 +85,7 @@ def __init__(
)
session_service = session_service or InMemorySessionService()
artifact_service = artifact_service or InMemoryArtifactService()
session_id_supplier = session_id_supplier or PreCreatedSessionIdSupplier(session_service)
self._metric_evaluator_registry = metric_evaluator_registry
self._session_service = session_service
self._artifact_service = artifact_service
Expand Down Expand Up @@ -394,7 +393,13 @@ async def _perform_inference_single_eval_item(
root_agent: BaseAgent,
) -> InferenceResult:
initial_session = eval_case.session_input
session_id = self._session_id_supplier()

session_id = await self._session_id_supplier.get_session_id(
app_name=app_name,
user_id=initial_session.user_id if initial_session else "test_user_id",
initial_state=initial_session.state if initial_session else {}
)

inference_result = InferenceResult(
app_name=app_name,
eval_set_id=eval_set_id,
Expand Down
11 changes: 11 additions & 0 deletions src/google/adk/sessions/base_session_id_supplier.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
from abc import ABC, abstractmethod
from typing import Any, Optional


class BaseSessionIdSupplier(ABC):
@abstractmethod
async def get_session_id(self, app_name: str, user_id: str, initial_state: Optional[dict[str, Any]] = None) -> str:
"""
Get a session id for a given app name and user id.
"""
pass
19 changes: 19 additions & 0 deletions src/google/adk/sessions/pre_created_session_id_supplier.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
from typing import Optional
from typing import Any
from typing_extensions import override
from google.adk.sessions.base_session_id_supplier import BaseSessionIdSupplier
from google.adk.sessions.base_session_service import BaseSessionService


class PreCreatedSessionIdSupplier(BaseSessionIdSupplier):
def __init__(self, session_service: BaseSessionService):
self.session_service = session_service

@override
async def get_session_id(self, app_name: Optional[str], user_id: Optional[str], initial_state: Optional[dict[str, Any]] = None) -> str:
created_session = await self.session_service.create_session(
app_name=app_name,
user_id=user_id,
state=initial_state
)
return created_session.id