From 8a857d1142efcf0859c8c7882c3405e7d2730353 Mon Sep 17 00:00:00 2001 From: Edoardo Rosa <6991986+notdodo@users.noreply.github.com> Date: Sun, 15 Nov 2026 12:01:58 +0100 Subject: [PATCH 1/2] docs: add reference docs, examples, release workflow, and Langfuse patch range - README: configuration table, per-module reference, limitations, and pointers to runnable examples under examples/, which the test suite compiles and runs offline. Taskfile lint and type checks cover them. - Release workflow: on v* tags, check the tag against the package version, run the checks, build and attest the wheel and sdist, and publish the GitHub release with notes from CHANGELOG.md. - Relax langfuse from ==5.2.0 to >=5.2.0,<5.3, and guard the private SDK internals the package relies on with focused contract tests. --- .github/workflows/release.yml | 68 ++++++ CHANGELOG.md | 12 + README.md | 218 +++++++++++++++++- Taskfile.yml | 4 +- examples/__init__.py | 1 + examples/basic_tracing.py | 43 ++++ examples/langchain_tool_callback.py | 48 ++++ examples/redaction.py | 52 +++++ pyproject.toml | 2 +- tests/package_contract/test_examples.py | 96 ++++++++ .../test_langfuse_internals.py | 103 +++++++++ uv.lock | 2 +- 12 files changed, 642 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 examples/__init__.py create mode 100644 examples/basic_tracing.py create mode 100644 examples/langchain_tool_callback.py create mode 100644 examples/redaction.py create mode 100644 tests/package_contract/test_examples.py create mode 100644 tests/package_contract/test_langfuse_internals.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..39e7d19 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,68 @@ +name: Release +on: + push: + tags: + - v* + +permissions: {} + +jobs: + release: + name: Build and publish GitHub release + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write + attestations: write + env: + UV_LOCKED: '1' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + python-version: '3.13' + enable-cache: false + - name: Install Task + uses: go-task/setup-task@a00fbb05ce67b35648be3c78cbc9fd85354c757e # v2.2.0 + with: + repo-token: ${{ github.token }} + - name: Check the tag matches the package version + env: + TAG: ${{ github.ref_name }} + run: | + version=$(uv version --short) + if [ "v${version}" != "${TAG}" ]; then + echo "::error::Tag ${TAG} does not match pyproject version ${version}" + exit 1 + fi + - name: Install dependencies + run: task install + - name: Format, lint, and type checks + run: task check + - name: Tests + run: task test + - name: Build + run: uv build + - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-path: dist/* + - name: Extract release notes from the changelog + env: + TAG: ${{ github.ref_name }} + run: | + awk -v version="${TAG#v}" ' + $0 ~ "^## \\[" version "\\]" { found = 1; next } + found && /^## / { exit } + found && !/^\[[^]]+\]: / { print } + ' CHANGELOG.md > release-notes.md + if ! grep -q '[^[:space:]]' release-notes.md; then + echo "::error::CHANGELOG.md has no section for ${TAG}" + exit 1 + fi + - name: Publish the GitHub release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.ref_name }} + run: gh release create "${TAG}" dist/* --verify-tag --title "${TAG}" --notes-file release-notes.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c7eabb9..43b556f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ All notable changes to this package are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses [Semantic Versioning](https://semver.org/). +## [Unreleased] + +### Added + +- README reference for every configuration field and public module, a limitations section, and runnable examples under `examples/`, compiled and run offline in the test suite. +- Tag-triggered release workflow that checks the tag against the package version, runs the checks, builds and attests the wheel and sdist, and publishes the GitHub release with notes from this changelog. + +### Changed + +- Relaxed the Langfuse requirement from an exact pin to the patch releases of the tested minor version (`langfuse>=5.2.0,<5.3`). + ## [0.4.0] - 2026-11-15 ### Added @@ -39,4 +50,5 @@ All notable changes to this package are documented here. The format follows [Kee - Replaced type-specific score convenience methods with the typed `Score` and `ScoreTarget` API. - Limited support to Python 3.13–3.14 and Langfuse 4.14.2. +[Unreleased]: https://github.com/DigintLab/digintlab-langfuse-python/compare/v0.4.0...HEAD [0.4.0]: https://github.com/DigintLab/digintlab-langfuse-python/releases/tag/v0.4.0 diff --git a/README.md b/README.md index ed453d9..2384daf 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,11 @@ Typed Python helpers for Langfuse tracing, scoring, callbacks, and LiteLLM metadata in DigIntLab agent runtimes. +Contents: [Install](#install) | [Usage](#usage) | [Configuration](#configuration) | [Modules](#modules) | [Sensitive data](#sensitive-data) | [Limitations](#limitations) | [Examples](#examples) | [Development](#development) | [API stability](#api-stability) | [License](#license) + ## Install -Requires Python 3.13, 3.14, or 3.15. Langfuse is pinned to the version tested by this package. +Requires Python 3.13, 3.14, or 3.15. Langfuse is constrained to the patch releases of the minor version tested by this package. The package is not published on PyPI. Install it from a tagged GitHub release (replace `v0.4.0` with the release you want): @@ -70,6 +72,189 @@ Important behavior: - Capture `telemetry.trace_runtime_context()` before sending work across a process or queue boundary. Pass it to `with_litellm_proxy_trace_metadata()` in the worker. - Plain HTTP endpoints require `allow_insecure_http=True`. +## Configuration + +`LangfuseClientConfig` is a frozen dataclass. Only `public_key` and `secret_key` are required. Invalid values raise `ValueError` at construction, before any SDK resource is allocated. + +| Field | Type | Default | Meaning | +| --- | --- | --- | --- | +| `public_key` | `str` | required | Langfuse project public key; must not be blank. | +| `secret_key` | `str` | required | Langfuse project secret key; must not be blank; hidden from `repr`. | +| `base_url` | `str \| None` | `None` | Langfuse host; must be an absolute HTTP(S) URL. When `None`, the `LANGFUSE_BASE_URL` or `LANGFUSE_HOST` environment variable is validated and used by the SDK. | +| `timeout` | `int \| None` | `None` | SDK request timeout; must be greater than zero. | +| `environment` | `str \| None` | `None` | Langfuse environment label. | +| `release` | `str \| None` | `None` | Release identifier attached to traces. | +| `sample_rate` | `float \| None` | `None` | Trace sampling rate between 0 and 1. | +| `flush_at` | `int \| None` | `None` | Batch size that triggers an export; at least 1. | +| `flush_interval` | `float \| None` | `None` | Seconds between exports; finite and greater than zero. | +| `debug` | `bool` | `False` | Enable Langfuse SDK debug logging. | +| `mask` | `MaskFunction \| None` | `None` | Custom Langfuse data mask. Mutually exclusive with `sensitive_data_redaction`. | +| `mask_otel_spans` | `MaskOtelSpansFunction \| None` | `None` | Custom export-stage span mask. Mutually exclusive with `sensitive_data_redaction`. | +| `httpx_client` | `httpx.Client \| None` | `None` | HTTP client the SDK uses to reach Langfuse. | +| `should_export_span` | `Callable[[ReadableSpan], bool] \| None` | `None` | Filter deciding which spans are exported. | +| `tracer_provider` | `TracerProvider \| None` | `None` | OpenTelemetry provider for the SDK and for opt-in instrumentation. | +| `span_exporter` | `SpanExporter \| None` | `None` | Replacement span exporter, for example an in-memory exporter in tests. | +| `redact_http_auth_headers` | `bool` | `False` | Redact the `Authorization` header on HTTPX spans; needs `instrument_httpx=True`. | +| `sensitive_data_redaction` | `SensitiveDataRedactionConfig \| None` | `None` | Install the built-in redaction policy as both mask hooks. | +| `allow_insecure_http` | `bool` | `False` | Permit a plain `http://` endpoint. | +| `flush_timeout_seconds` | `float` | `30.0` | Time limit for `flush()`; finite and greater than zero. | +| `shutdown_timeout_seconds` | `float` | `30.0` | Time limit for `shutdown()`; finite and greater than zero. | + +`init_langfuse_telemetry(config, *, instrument_httpx=False, instrument_botocore=False)` creates the client and returns a `LangfuseTelemetry`. + +## Modules + +Every name below is in the module's `__all__`. Examples assume `telemetry` from the Usage section. + +### `client` and top-level exports + +`LangfuseTelemetry` owns the SDK client (`telemetry.client`) plus optional instrumentation. Use it as an async context manager, or call `await telemetry.flush()` and `await telemetry.shutdown()`. `trace_call` propagates session and user ids and, with `root_observation=True`, opens a root observation of kind `root_kind` (default `ObservationKind.AGENT`). It also accepts `tags`, `metadata`, `version`, `trace_name`, `trace_id`, `parent_span_id`, `root_input`, and `root_metadata`. Other methods wrap the module functions below with this owner's client: `observe_node_span`, `emit_guardrail_observation`, `tool_span_callback`, `langchain_callbacks`, `with_litellm_proxy_trace_metadata`, `trace_runtime_context`, `emit_score`, `score_current_trace`, `score_current_span`, `instrument`, and `uninstrument`. There is no separate `trace_scope`; `trace_call` is the trace scope. + +```python +from digintlab_langfuse import ObservationKind + +with telemetry.trace_call( + name="ingest", + session_id="s-1", + user_id="u-1", + root_observation=True, + root_kind=ObservationKind.CHAIN, + tags=["batch"], +): + ... +``` + +### `spans` + +Observation helpers over context-local active-parent state: `observe_node_span`, `observe_generation`, `emit_guardrail_observation`, and `current_trace_id`. `trace_runtime_context` returns a `TraceRuntimeContext` for work that crosses a process or queue boundary. `scrub_auth_header` and `scrub_auth_header_async` are the HTTPX hooks behind `redact_http_auth_headers`. All helpers open observations only inside an active trace. + +```python +from digintlab_langfuse.spans import ObservationKind, observe_generation, observe_node_span + +with observe_node_span("retrieve", enabled=True, kind=ObservationKind.RETRIEVER, client_getter=lambda: telemetry.client): + with observe_generation("answer", model="my-model", client_getter=lambda: telemetry.client) as generation: + if generation is not None: + generation.update(output="done") +``` + +### `callbacks` + +`ToolSpanCallback` is a LangChain callback handler that records each tool call as a Langfuse tool observation under the active node span. `summarize_output` returns a `ToolObservationResult` (`output`, `level`, `status_message`) to control what is stored. The tool input is always captured. + +```python +from digintlab_langfuse.callbacks import ToolObservationResult + + +def summarize(output: object) -> ToolObservationResult: + return {"output": {"status": "ok", "chars": len(str(output))}} + + +callback = telemetry.tool_span_callback(summarize_output=summarize) +``` + +### `litellm` + +`with_litellm_proxy_trace_metadata` merges the active trace ids into `extra_body.metadata` of LiteLLM proxy kwargs, and returns the kwargs unchanged when no trace is active. Pass `trace_context=` to use a captured `TraceRuntimeContext` instead of the active trace. It raises `TypeError` if `extra_body` is not a dict. + +```python +from digintlab_langfuse.litellm import with_litellm_proxy_trace_metadata + +kwargs = with_litellm_proxy_trace_metadata({"model": "my-model"}, trace_context=telemetry.trace_runtime_context()) +``` + +### `scores` + +One typed API for all score kinds: `NumericScore`, `BooleanScore`, `CategoricalScore`, `TextScore`, and `CorrectionScore`. `ScoreTarget` selects the trace, observation, or session (`run_id`, `trace_id`, `observation_id`, `session_id`, `dataset_run_id`, `score_id`, `comment`, `config_id`, `metadata`). `score_elapsed` measures a block and emits its seconds as a numeric score. + +```python +from digintlab_langfuse.scores import BooleanScore, ScoreTarget, emit_score, score_elapsed + +emit_score(telemetry.client, score=BooleanScore(name="passed", value=True), target=ScoreTarget(trace_id=trace_id)) +with score_elapsed(telemetry.client, name="latency_seconds", target=ScoreTarget(trace_id=trace_id), digits=3): + ... +``` + +### `prompts` + +Typed requests for fetching and creating managed prompts. `TextPromptRequest` and `ChatPromptRequest` take `name`, `version`, `label`, `cache_ttl_seconds`, `fallback`, `max_retries`, and `fetch_timeout_seconds`. `TextManagedPrompt` and `ChatManagedPrompt` take `name`, `prompt`, `labels`, `tags`, `config`, and `commit_message`. `create_or_update_prompt` creates a new prompt version. + +```python +from digintlab_langfuse.prompts import TextManagedPrompt, TextPromptRequest, create_or_update_prompt, get_prompt + +create_or_update_prompt(telemetry.client, TextManagedPrompt(name="greeting", prompt="Hello {{name}}", labels=("production",))) +prompt = get_prompt(telemetry.client, TextPromptRequest(name="greeting", label="production")) +``` + +### `experiments` + +Thin helpers over Langfuse datasets and local experiments. `run_local_experiment` normalizes items to JSON values and takes `evaluators`, `run_evaluators`, `max_concurrency` (default 50), `metadata`, and `experiment_id`. + +```python +from digintlab_langfuse.experiments import DatasetConfig, create_dataset, run_local_experiment + +create_dataset(telemetry.client, DatasetConfig(name="smoke")) +result = run_local_experiment( + telemetry.client, + name="smoke-run", + data=[{"input": "2+2", "expected_output": "4"}], + task=lambda *, item, **kwargs: "4", +) +``` + +`create_dataset_item` and the `ExperimentItem` type are also exported. + +### `budgets` + +Numeric ceiling checks for CI-style gates. `check_numeric_budgets` returns a `NumericBudgetViolation` for each missing or exceeded measurement. `validate_numeric_budget_ceilings` validates raw ceilings (optionally against `allowed_names`) and rejects non-numeric or unknown entries. + +```python +from digintlab_langfuse.budgets import check_numeric_budgets, validate_numeric_budget_ceilings + +ceilings = validate_numeric_budget_ceilings({"p95_seconds": 2.0}) +for violation in check_numeric_budgets({"p95_seconds": 3.1}, ceilings): + print(violation.name, violation.message) # p95_seconds 3.1 > 2.0 +``` + +### `timing` + +`elapsed_timer` times a local block. `create_span_timing_recorder` adds a `SpanTimingProcessor` to a tracer provider (a new one by default) and aggregates ended spans into `SpanTimingBucket` values keyed by `default_span_timing_key` or your own `key`. Beyond `max_buckets` (default 1000), new keys fold into `__overflow__`. Pass `recorder.tracer_provider` as `LangfuseClientConfig.tracer_provider` to record Langfuse spans. + +```python +from digintlab_langfuse.timing import create_span_timing_recorder, elapsed_timer + +with elapsed_timer() as timer: + ... +print(timer.rounded(3)) + +recorder = create_span_timing_recorder() +with recorder.tracer_provider.get_tracer("demo").start_as_current_span("work"): + ... +buckets = recorder.snapshot(reset=True) +``` + +### `redaction` + +Opt-in masking built from a `SensitiveDataRedactionConfig` (`replacement="[REDACTED]"`, `sensitive_field_names`, `pii_entities`, `pii_score_threshold=0.4`, `redact_bearer_tokens=True`). `create_sensitive_data_mask` and `create_sensitive_data_otel_mask` build the two Langfuse hooks; `redact_sensitive_data` applies the policy to any value directly. See [Sensitive data](#sensitive-data). + +```python +from digintlab_langfuse import SensitiveDataRedactionConfig +from digintlab_langfuse.redaction import redact_sensitive_data + +policy = SensitiveDataRedactionConfig(pii_entities=frozenset({"EMAIL_ADDRESS"})) +redact_sensitive_data({"password": "hunter2", "note": "mail a@example.com"}, policy) +# {"password": "[REDACTED]", "note": "mail [REDACTED]"} +``` + +### `json_values` + +`JsonValue` is the recursive JSON type used across the API. `to_json_value` converts enums, mappings, sequences, and other objects (via `str`) into it, and raises `ValueError` for non-finite floats. + +```python +from digintlab_langfuse.json_values import JsonValue, to_json_value + +payload: JsonValue = to_json_value({"count": 3, "tags": ("a", "b")}) +``` + ## Sensitive data PII redaction is optional: @@ -102,7 +287,34 @@ The policy masks common credential fields and bearer tokens, then uses Presidio' > `SensitiveDataRedactionConfig` also does not sanitize metadata forwarded to LiteLLM. Keep runtime-context, run-manager, and request metadata non-sensitive, > or sanitize it before calling `with_litellm_proxy_trace_metadata()`. -HTTP authorization-header redaction is separate. Enable both `instrument_httpx=True` and `redact_http_auth_headers=True`. Tool callbacks capture inputs and outputs unless you provide `summarize_output` or enable masking. +HTTP authorization-header redaction is separate. Enable both `instrument_httpx=True` and `redact_http_auth_headers=True`. Tool callbacks always capture the tool input and capture the output unless you provide `summarize_output`; enable masking to cover both. + +## Limitations + +- **One Langfuse project per process.** A second `init_langfuse_telemetry()` raises `RuntimeError`, whether it reuses the public key or uses another project's. It also raises when a Langfuse SDK resource was created outside this package. `shutdown()` does not release this ownership, so initialize once per process lifetime. Run separate projects in separate processes. +- **Langfuse is limited to one minor version** (`langfuse>=5.2.0,<5.3`). The package reaches into SDK internals, which `tests/package_contract/test_langfuse_internals.py` checks on every dependency update, so a new Langfuse minor release is only allowed after review. Do not override the range. +- **Private SDK internals are used.** Ownership checks and lifecycle handling read `LangfuseResourceManager` state (`_instances`, `_lock`) and the client's `_resources`. `ToolSpanCallback` is typed against `LangfuseObservationWrapper` from `langfuse._client.span`. +- **Observation helpers fail open and no-op quietly.** Errors in `trace_call`, `observe_node_span`, `observe_generation`, `emit_guardrail_observation`, `ToolSpanCallback`, and `with_litellm_proxy_trace_metadata` are logged and never replace your result or exception. Outside an active trace these helpers record nothing. Configuration, scoring, `flush`, and `shutdown` fail fast. +- **Instrumentation is process-wide.** HTTPX and botocore instrumentors are shared by every owner in the process and must use the same tracer provider and redaction setting, or initialization raises. Instrumentation installed by other code is left in place, and HTTPX initialization fails if that code owns it and `redact_http_auth_headers=True`. `shutdown()` uninstruments only what this package installed. +- **Redaction has limits.** + - It runs through Langfuse's `mask` and `mask_otel_spans` hooks, so it covers masked data and string span attributes only. Span names, identifiers, links, events, and resource attributes are not rewritten. + - Metadata forwarded to LiteLLM is not sanitized. + - Matching is by field name (normalized, so `apiKey` matches `api_key`), bearer-token pattern, and Presidio's English rule-based recognizers. There is no NLP model, so names and addresses are not detected, and non-English text is not analyzed. + - Secrets in free text without a recognized field name or bearer prefix are not masked. + - `redact_http_auth_headers` only masks `Authorization`; other headers such as cookies or API-key headers are not scrubbed. + - The default `pii_entities` apply when you enable `sensitive_data_redaction`; pass an empty set to skip PII analysis and the `redaction` extra. + +## Examples + +Runnable scripts live in [`examples/`](examples). Each reads `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, and `LANGFUSE_BASE_URL`, and exits with a message if one is unset: + +- [`basic_tracing.py`](examples/basic_tracing.py): trace one call and attach numeric, categorical, and elapsed-time scores. +- [`langchain_tool_callback.py`](examples/langchain_tool_callback.py): record a LangChain tool call as a tool observation. +- [`redaction.py`](examples/redaction.py): enable the built-in redaction policy. + +```bash +uv run python examples/basic_tracing.py +``` ## Development @@ -129,7 +341,7 @@ The public API is the names listed in each public module's `__all__` plus the to Versioning follows [SemVer](https://semver.org/). Before 1.0, a minor version (0.x to 0.(x+1)) may contain breaking changes, which are listed in `CHANGELOG.md` under "Changed" or "Removed". Patch versions are backward compatible. -Langfuse is pinned to an exact tested version, and a pin change is released as at least a minor version. +Langfuse is limited to the patch releases of one tested minor version. Moving to a new Langfuse minor version is released as at least a minor version of this package. ## License diff --git a/Taskfile.yml b/Taskfile.yml index 9bb9795..9081618 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -28,12 +28,12 @@ tasks: lint: desc: Run lint checks cmds: - - uv run ruff check src/ tests/ + - uv run ruff check src/ tests/ examples/ type-check: desc: Run static type checks cmds: - - uv run mypy src/ tests/ + - uv run mypy src/ tests/ examples/ check: desc: Run formatting, lint, and type checks diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 0000000..1f7a088 --- /dev/null +++ b/examples/__init__.py @@ -0,0 +1 @@ +"""Runnable usage examples for digintlab-langfuse-python.""" diff --git a/examples/basic_tracing.py b/examples/basic_tracing.py new file mode 100644 index 0000000..6fc0455 --- /dev/null +++ b/examples/basic_tracing.py @@ -0,0 +1,43 @@ +"""Trace one call and attach scores. + +Run with LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY, and LANGFUSE_BASE_URL set. +""" + +import asyncio +import os +import sys + +from digintlab_langfuse import LangfuseClientConfig, init_langfuse_telemetry +from digintlab_langfuse.scores import CategoricalScore, NumericScore, ScoreTarget, score_elapsed + + +def config_from_env() -> LangfuseClientConfig: + """Build a client config from the environment, exiting with a clear message when it is incomplete.""" + names = ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_BASE_URL") + missing = [name for name in names if not os.environ.get(name)] + if missing: + sys.exit(f"Set these environment variables before running this example: {', '.join(missing)}") + return LangfuseClientConfig( + public_key=os.environ["LANGFUSE_PUBLIC_KEY"], + secret_key=os.environ["LANGFUSE_SECRET_KEY"], + base_url=os.environ["LANGFUSE_BASE_URL"], + ) + + +async def run(config: LangfuseClientConfig) -> None: + """Open a root observation, score the active trace, then emit scores against an explicit trace id.""" + telemetry = init_langfuse_telemetry(config) + async with telemetry: + with telemetry.trace_call(name="example-run", session_id="example-session", user_id="example-user", root_observation=True): + telemetry.score_current_trace(score=NumericScore(name="quality", value=0.95)) + runtime_context = telemetry.trace_runtime_context() + if runtime_context is not None: + target = ScoreTarget(trace_id=runtime_context.trace_id) + with score_elapsed(telemetry.client, name="work_seconds", target=target): + await asyncio.sleep(0.01) + telemetry.emit_score(score=CategoricalScore(name="outcome", value="ok"), target=target) + await telemetry.flush() + + +if __name__ == "__main__": + asyncio.run(run(config_from_env())) diff --git a/examples/langchain_tool_callback.py b/examples/langchain_tool_callback.py new file mode 100644 index 0000000..4546b4c --- /dev/null +++ b/examples/langchain_tool_callback.py @@ -0,0 +1,48 @@ +"""Record a LangChain tool call as a Langfuse tool observation. + +Run with LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY, and LANGFUSE_BASE_URL set. +""" + +import asyncio +import os +import sys + +from langchain_core.tools import tool + +from digintlab_langfuse import LangfuseClientConfig, ObservationKind, init_langfuse_telemetry + + +@tool +def add(left: int, right: int) -> int: + """Add two integers.""" + return left + right + + +def config_from_env() -> LangfuseClientConfig: + """Build a client config from the environment, exiting with a clear message when it is incomplete.""" + names = ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_BASE_URL") + missing = [name for name in names if not os.environ.get(name)] + if missing: + sys.exit(f"Set these environment variables before running this example: {', '.join(missing)}") + return LangfuseClientConfig( + public_key=os.environ["LANGFUSE_PUBLIC_KEY"], + secret_key=os.environ["LANGFUSE_SECRET_KEY"], + base_url=os.environ["LANGFUSE_BASE_URL"], + ) + + +async def run(config: LangfuseClientConfig) -> int: + """Invoke a tool inside a node observation so the callback has an active parent to attach to.""" + telemetry = init_langfuse_telemetry(config) + async with telemetry: + with ( + telemetry.trace_call(name="example-tools", session_id="example-session", user_id="example-user", root_observation=True), + telemetry.observe_node_span("tools-node", enabled=True, kind=ObservationKind.CHAIN), + ): + result: int = add.invoke({"left": 2, "right": 3}, config={"callbacks": [telemetry.tool_span_callback()]}) + await telemetry.flush() + return result + + +if __name__ == "__main__": + sys.stdout.write(f"add(2, 3) = {asyncio.run(run(config_from_env()))}\n") diff --git a/examples/redaction.py b/examples/redaction.py new file mode 100644 index 0000000..792f684 --- /dev/null +++ b/examples/redaction.py @@ -0,0 +1,52 @@ +"""Mask credentials and PII before data leaves the process. + +Requires the `redaction` extra. Run with LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY, and LANGFUSE_BASE_URL set. +""" + +import asyncio +import os +import sys + +from digintlab_langfuse import LangfuseClientConfig, SensitiveDataRedactionConfig, init_langfuse_telemetry +from digintlab_langfuse.json_values import JsonValue +from digintlab_langfuse.redaction import redact_sensitive_data + + +def redaction_policy() -> SensitiveDataRedactionConfig: + """Return the example policy: default credential fields plus email and phone PII.""" + return SensitiveDataRedactionConfig(pii_entities=frozenset({"EMAIL_ADDRESS", "PHONE_NUMBER"})) + + +def config_from_env() -> LangfuseClientConfig: + """Build a client config with redaction enabled, exiting with a clear message when the environment is incomplete.""" + names = ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_BASE_URL") + missing = [name for name in names if not os.environ.get(name)] + if missing: + sys.exit(f"Set these environment variables before running this example: {', '.join(missing)}") + return LangfuseClientConfig( + public_key=os.environ["LANGFUSE_PUBLIC_KEY"], + secret_key=os.environ["LANGFUSE_SECRET_KEY"], + base_url=os.environ["LANGFUSE_BASE_URL"], + sensitive_data_redaction=redaction_policy(), + ) + + +async def run(config: LangfuseClientConfig) -> JsonValue: + """Trace a call whose input contains sensitive values, and return what the policy does to the same payload.""" + payload: dict[str, JsonValue] = {"api_key": "sk-secret", "note": "Contact jane.doe@example.com"} + telemetry = init_langfuse_telemetry(config) + async with telemetry: + with telemetry.trace_call( + name="example-redaction", + session_id="example-session", + user_id="example-user", + root_observation=True, + root_input=payload, + ): + pass + await telemetry.flush() + return redact_sensitive_data(payload, redaction_policy()) + + +if __name__ == "__main__": + sys.stdout.write(f"{asyncio.run(run(config_from_env()))}\n") diff --git a/pyproject.toml b/pyproject.toml index 24e28d1..1fbe624 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "botocore>=1.43.82,<2", "httpx>=0.28.1,<1", "langchain-core>=1.6.1,<2", - "langfuse==5.2.0", + "langfuse>=5.2.0,<5.3", "opentelemetry-api>=1.44.0,<2", "opentelemetry-instrumentation-botocore>=0.65b0,<1", "opentelemetry-instrumentation-httpx>=0.65b0,<1", diff --git a/tests/package_contract/test_examples.py b/tests/package_contract/test_examples.py new file mode 100644 index 0000000..7da311a --- /dev/null +++ b/tests/package_contract/test_examples.py @@ -0,0 +1,96 @@ +"""Offline checks that keep the scripts in `examples/` importable and runnable.""" + +import importlib.util +import py_compile +from pathlib import Path +from types import ModuleType +from uuid import uuid4 + +import httpx +import pytest +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from digintlab_langfuse import LangfuseClientConfig, SensitiveDataRedactionConfig + +EXAMPLES_DIR = Path(__file__).resolve().parents[2] / "examples" +EXAMPLE_PATHS = sorted(path for path in EXAMPLES_DIR.glob("*.py") if path.name != "__init__.py") +ENV_NAMES = ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_BASE_URL") + + +def _load(path: Path) -> ModuleType: + spec = importlib.util.spec_from_file_location(f"_example_{path.stem}", path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _offline_config(http_client: httpx.Client, sensitive_data_redaction: SensitiveDataRedactionConfig | None = None) -> LangfuseClientConfig: + return LangfuseClientConfig( + public_key=f"pk-example-{uuid4()}", + secret_key="test-secret", + httpx_client=http_client, + span_exporter=InMemorySpanExporter(), + flush_at=100, + flush_interval=60.0, + sensitive_data_redaction=sensitive_data_redaction, + ) + + +def test_examples_exist() -> None: + assert {path.stem for path in EXAMPLE_PATHS} == {"basic_tracing", "langchain_tool_callback", "redaction"} + + +@pytest.mark.parametrize("path", EXAMPLE_PATHS, ids=lambda path: path.stem) +def test_example_compiles_and_exposes_entry_points(path: Path, tmp_path: Path) -> None: + py_compile.compile(str(path), cfile=str(tmp_path / f"{path.stem}.pyc"), doraise=True) + module = _load(path) + assert callable(module.run) + assert callable(module.config_from_env) + + +@pytest.mark.parametrize("path", EXAMPLE_PATHS, ids=lambda path: path.stem) +def test_example_exits_with_clear_message_without_credentials(path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + for name in ENV_NAMES: + monkeypatch.delenv(name, raising=False) + module = _load(path) + with pytest.raises(SystemExit) as exit_info: + module.config_from_env() + message = str(exit_info.value) + assert all(name in message for name in ENV_NAMES) + + +@pytest.mark.parametrize("path", EXAMPLE_PATHS, ids=lambda path: path.stem) +def test_example_reads_credentials_from_environment(path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-env") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-env") + monkeypatch.setenv("LANGFUSE_BASE_URL", "https://langfuse.example.com") + config = _load(path).config_from_env() + assert isinstance(config, LangfuseClientConfig) + assert config.public_key == "pk-env" + assert config.base_url == "https://langfuse.example.com" + + +# The pinned Langfuse SDK calls archived backoff 2.2.1 from its score worker, which uses a deprecated asyncio helper on Python 3.14+. +@pytest.mark.filterwarnings("ignore:'asyncio.iscoroutinefunction' is deprecated:DeprecationWarning:backoff._decorator") +@pytest.mark.asyncio +async def test_basic_tracing_example_runs_offline() -> None: + module = _load(EXAMPLES_DIR / "basic_tracing.py") + with httpx.Client(transport=httpx.MockTransport(lambda _request: httpx.Response(202, json={"message": "Accepted"}))) as http_client: + await module.run(_offline_config(http_client)) + + +@pytest.mark.asyncio +async def test_langchain_tool_callback_example_runs_offline() -> None: + module = _load(EXAMPLES_DIR / "langchain_tool_callback.py") + with httpx.Client(transport=httpx.MockTransport(lambda _request: httpx.Response(202, json={"message": "Accepted"}))) as http_client: + assert await module.run(_offline_config(http_client)) == 5 + + +@pytest.mark.asyncio +async def test_redaction_example_runs_offline() -> None: + module = _load(EXAMPLES_DIR / "redaction.py") + with httpx.Client(transport=httpx.MockTransport(lambda _request: httpx.Response(202, json={"message": "Accepted"}))) as http_client: + redacted = await module.run(_offline_config(http_client, module.redaction_policy())) + assert redacted == {"api_key": "[REDACTED]", "note": "Contact [REDACTED]"} diff --git a/tests/package_contract/test_langfuse_internals.py b/tests/package_contract/test_langfuse_internals.py new file mode 100644 index 0000000..f3c67a2 --- /dev/null +++ b/tests/package_contract/test_langfuse_internals.py @@ -0,0 +1,103 @@ +"""Contract tests for the private Langfuse SDK internals this package relies on. + +Each failure names the attribute, the dependent module, and the required action so a +Langfuse version bump surfaces breakage here instead of at runtime. +""" + +import importlib +import threading +from uuid import uuid4 + +from langfuse import Langfuse, get_client +from langfuse._client.resource_manager import LangfuseResourceManager +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from digintlab_langfuse import LangfuseClientConfig +from digintlab_langfuse.client import _init_langfuse_client + +_UPGRADE_HINT = "update digintlab_langfuse before bumping langfuse" + + +def _message(attribute: str, module: str, expectation: str) -> str: + return f"{attribute} {expectation}; digintlab_langfuse.{module} depends on it: {_UPGRADE_HINT}." + + +def _build_client() -> tuple[str, Langfuse]: + public_key = f"pk-internals-{uuid4()}" + client = _init_langfuse_client( + LangfuseClientConfig( + public_key=public_key, + secret_key="internals-secret", + span_exporter=InMemorySpanExporter(), + ) + ) + return public_key, client + + +def test_resource_manager_lock_is_a_reentrant_context_manager() -> None: + lock = LangfuseResourceManager._lock + expectation = "must be a reentrant lock usable as a context manager (initialize_client holds it while Langfuse() re-acquires it)" + message = _message("LangfuseResourceManager._lock", "_lifecycle:58", expectation) + + assert isinstance(lock, type(threading.RLock())), message + with lock, lock: + pass + + +def test_resource_manager_instances_is_a_dict_keyed_by_public_key() -> None: + public_key, _client = _build_client() + instances = LangfuseResourceManager._instances + + assert isinstance(instances, dict), _message("LangfuseResourceManager._instances", "_lifecycle:74", "must be a dict") + assert public_key in instances, _message("LangfuseResourceManager._instances", "_lifecycle:83", "must be keyed by the client's public key") + assert isinstance(instances[public_key], LangfuseResourceManager), _message( + "LangfuseResourceManager._instances", "_lifecycle:83", "must map public keys to LangfuseResourceManager" + ) + + +def test_resource_manager_instances_supports_key_deletion() -> None: + public_key, _client = _build_client() + instances = LangfuseResourceManager._instances + + with LangfuseResourceManager._lock: + del instances[public_key] + + assert not instances, _message("LangfuseResourceManager._instances", "_lifecycle:84", "must support `del` by public key") + + +def test_langfuse_resources_attribute_is_the_shared_resource_manager() -> None: + public_key, client = _build_client() + alias = get_client(public_key=public_key) + + assert hasattr(client, "_resources"), _message("Langfuse._resources", "_lifecycle:38 and spans:131", "must exist on Langfuse") + assert isinstance(client._resources, LangfuseResourceManager), _message( + "Langfuse._resources", "_lifecycle:38 and spans:131", "must be a LangfuseResourceManager" + ) + assert alias is not client, _message("get_client()", "_lifecycle:38", "must return a distinct facade for an existing key") + assert alias._resources is client._resources, _message( + "Langfuse._resources", "_lifecycle:38 and spans:131", "must be shared by every facade of one public key" + ) + + +def test_resource_manager_shutdown_flag_and_method() -> None: + _public_key, client = _build_client() + resources = client._resources + + assert resources is not None + flag = resources._shutdown + assert flag is False, _message("LangfuseResourceManager._shutdown", "tests/conftest.py cleanup", "must be a bool, False for a live client") + assert callable(resources.shutdown), _message("LangfuseResourceManager.shutdown", "tests/conftest.py cleanup", "must be callable") + + resources.shutdown() + + assert resources._shutdown is True, _message( + "LangfuseResourceManager._shutdown", "tests/conftest.py cleanup", "must become True after shutdown()" + ) + + +def test_langfuse_observation_wrapper_is_importable_from_private_module() -> None: + module = importlib.import_module("langfuse._client.span") + + assert hasattr(module, "LangfuseObservationWrapper"), _message( + "langfuse._client.span.LangfuseObservationWrapper", "callbacks:31", "must be importable" + ) diff --git a/uv.lock b/uv.lock index ce562bc..a60034d 100644 --- a/uv.lock +++ b/uv.lock @@ -683,7 +683,7 @@ requires-dist = [ { name = "botocore", specifier = ">=1.43.82,<2" }, { name = "httpx", specifier = ">=0.28.1,<1" }, { name = "langchain-core", specifier = ">=1.6.1,<2" }, - { name = "langfuse", specifier = "==5.2.0" }, + { name = "langfuse", specifier = ">=5.2.0,<5.3" }, { name = "opentelemetry-api", specifier = ">=1.44.0,<2" }, { name = "opentelemetry-instrumentation-botocore", specifier = ">=0.65b0,<1" }, { name = "opentelemetry-instrumentation-httpx", specifier = ">=0.65b0,<1" }, From 9051dd99384e65e41c78374ae0d0274ed9ba53e3 Mon Sep 17 00:00:00 2001 From: Edoardo Rosa <6991986+notdodo@users.noreply.github.com> Date: Sun, 15 Nov 2026 12:04:30 +0100 Subject: [PATCH 2/2] docs: drop the examples and shorten the README --- CHANGELOG.md | 2 +- README.md | 320 +++--------------------- Taskfile.yml | 4 +- examples/__init__.py | 1 - examples/basic_tracing.py | 43 ---- examples/langchain_tool_callback.py | 48 ---- examples/redaction.py | 52 ---- tests/package_contract/test_examples.py | 96 ------- 8 files changed, 39 insertions(+), 527 deletions(-) delete mode 100644 examples/__init__.py delete mode 100644 examples/basic_tracing.py delete mode 100644 examples/langchain_tool_callback.py delete mode 100644 examples/redaction.py delete mode 100644 tests/package_contract/test_examples.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 43b556f..e771c0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ All notable changes to this package are documented here. The format follows [Kee ### Added -- README reference for every configuration field and public module, a limitations section, and runnable examples under `examples/`, compiled and run offline in the test suite. +- Shorter README with a module overview and a limitations section. - Tag-triggered release workflow that checks the tag against the package version, runs the checks, builds and attests the wheel and sdist, and publishes the GitHub release with notes from this changelog. ### Changed diff --git a/README.md b/README.md index 2384daf..6745d89 100644 --- a/README.md +++ b/README.md @@ -1,269 +1,54 @@ # digintlab-langfuse-python -Typed Python helpers for Langfuse tracing, scoring, callbacks, and LiteLLM metadata in DigIntLab agent runtimes. - -Contents: [Install](#install) | [Usage](#usage) | [Configuration](#configuration) | [Modules](#modules) | [Sensitive data](#sensitive-data) | [Limitations](#limitations) | [Examples](#examples) | [Development](#development) | [API stability](#api-stability) | [License](#license) +Typed helpers for tracing, scoring, and redacting agent runs with Langfuse. ## Install -Requires Python 3.13, 3.14, or 3.15. Langfuse is constrained to the patch releases of the minor version tested by this package. - -The package is not published on PyPI. Install it from a tagged GitHub release (replace `v0.4.0` with the release you want): +Python 3.13 to 3.15. Not on PyPI; install a tagged release from GitHub: ```bash uv add "digintlab-langfuse-python @ git+https://github.com/DigintLab/digintlab-langfuse-python@v0.4.0" ``` -With pip: - -```bash -pip install "digintlab-langfuse-python @ git+https://github.com/DigintLab/digintlab-langfuse-python@v0.4.0" -``` - -Each [GitHub release](https://github.com/DigintLab/digintlab-langfuse-python/releases) also attaches a wheel and a source distribution, which can be installed directly: - -```bash -uv add "digintlab-langfuse-python @ https://github.com/DigintLab/digintlab-langfuse-python/releases/download/v0.4.0/digintlab_langfuse_python-0.4.0-py3-none-any.whl" -``` +Add the `[redaction]` extra for PII detection. Each [release](https://github.com/DigintLab/digintlab-langfuse-python/releases) also ships a wheel and an sdist. ## Usage ```python from digintlab_langfuse import LangfuseClientConfig, init_langfuse_telemetry +from digintlab_langfuse.scores import NumericScore, ScoreTarget telemetry = init_langfuse_telemetry( - LangfuseClientConfig( - public_key=public_key, - secret_key=secret_key, - environment="production", - release=release, - redact_http_auth_headers=True, - ), - instrument_httpx=True, + LangfuseClientConfig(public_key=public_key, secret_key=secret_key, environment="production"), ) async with telemetry: - with telemetry.trace_call( - name="agent-run", - session_id=session_id, - user_id=user_id, - root_observation=True, - ): + with telemetry.trace_call(name="agent-run", session_id=session_id, user_id=user_id, root_observation=True): await run_agent() -``` - -Scores use one typed API rather than type-specific convenience methods: - -```python -from digintlab_langfuse.scores import NumericScore, ScoreTarget - -telemetry.emit_score( - score=NumericScore(name="quality", value=0.95), - target=ScoreTarget(trace_id=trace_id), -) -``` - -Important behavior: - -- Create one telemetry owner per process, after workers fork. Do not also construct `Langfuse(...)` directly. -- HTTPX and botocore instrumentation are opt-in. Instrumentation owned by other libraries is left untouched; initialization fails when HTTPX is externally instrumented and authorization-header redaction is requested because the redaction hooks cannot be guaranteed. -- Observation helpers fail open so telemetry errors do not replace application results or exceptions. Configuration, scoring, flush, and shutdown fail fast. -- Use `await telemetry.flush()` to export pending data without closing the owner. The async context manager shuts it down. -- Capture `telemetry.trace_runtime_context()` before sending work across a process or queue boundary. Pass it to `with_litellm_proxy_trace_metadata()` in the worker. -- Plain HTTP endpoints require `allow_insecure_http=True`. - -## Configuration - -`LangfuseClientConfig` is a frozen dataclass. Only `public_key` and `secret_key` are required. Invalid values raise `ValueError` at construction, before any SDK resource is allocated. - -| Field | Type | Default | Meaning | -| --- | --- | --- | --- | -| `public_key` | `str` | required | Langfuse project public key; must not be blank. | -| `secret_key` | `str` | required | Langfuse project secret key; must not be blank; hidden from `repr`. | -| `base_url` | `str \| None` | `None` | Langfuse host; must be an absolute HTTP(S) URL. When `None`, the `LANGFUSE_BASE_URL` or `LANGFUSE_HOST` environment variable is validated and used by the SDK. | -| `timeout` | `int \| None` | `None` | SDK request timeout; must be greater than zero. | -| `environment` | `str \| None` | `None` | Langfuse environment label. | -| `release` | `str \| None` | `None` | Release identifier attached to traces. | -| `sample_rate` | `float \| None` | `None` | Trace sampling rate between 0 and 1. | -| `flush_at` | `int \| None` | `None` | Batch size that triggers an export; at least 1. | -| `flush_interval` | `float \| None` | `None` | Seconds between exports; finite and greater than zero. | -| `debug` | `bool` | `False` | Enable Langfuse SDK debug logging. | -| `mask` | `MaskFunction \| None` | `None` | Custom Langfuse data mask. Mutually exclusive with `sensitive_data_redaction`. | -| `mask_otel_spans` | `MaskOtelSpansFunction \| None` | `None` | Custom export-stage span mask. Mutually exclusive with `sensitive_data_redaction`. | -| `httpx_client` | `httpx.Client \| None` | `None` | HTTP client the SDK uses to reach Langfuse. | -| `should_export_span` | `Callable[[ReadableSpan], bool] \| None` | `None` | Filter deciding which spans are exported. | -| `tracer_provider` | `TracerProvider \| None` | `None` | OpenTelemetry provider for the SDK and for opt-in instrumentation. | -| `span_exporter` | `SpanExporter \| None` | `None` | Replacement span exporter, for example an in-memory exporter in tests. | -| `redact_http_auth_headers` | `bool` | `False` | Redact the `Authorization` header on HTTPX spans; needs `instrument_httpx=True`. | -| `sensitive_data_redaction` | `SensitiveDataRedactionConfig \| None` | `None` | Install the built-in redaction policy as both mask hooks. | -| `allow_insecure_http` | `bool` | `False` | Permit a plain `http://` endpoint. | -| `flush_timeout_seconds` | `float` | `30.0` | Time limit for `flush()`; finite and greater than zero. | -| `shutdown_timeout_seconds` | `float` | `30.0` | Time limit for `shutdown()`; finite and greater than zero. | - -`init_langfuse_telemetry(config, *, instrument_httpx=False, instrument_botocore=False)` creates the client and returns a `LangfuseTelemetry`. - -## Modules - -Every name below is in the module's `__all__`. Examples assume `telemetry` from the Usage section. - -### `client` and top-level exports - -`LangfuseTelemetry` owns the SDK client (`telemetry.client`) plus optional instrumentation. Use it as an async context manager, or call `await telemetry.flush()` and `await telemetry.shutdown()`. `trace_call` propagates session and user ids and, with `root_observation=True`, opens a root observation of kind `root_kind` (default `ObservationKind.AGENT`). It also accepts `tags`, `metadata`, `version`, `trace_name`, `trace_id`, `parent_span_id`, `root_input`, and `root_metadata`. Other methods wrap the module functions below with this owner's client: `observe_node_span`, `emit_guardrail_observation`, `tool_span_callback`, `langchain_callbacks`, `with_litellm_proxy_trace_metadata`, `trace_runtime_context`, `emit_score`, `score_current_trace`, `score_current_span`, `instrument`, and `uninstrument`. There is no separate `trace_scope`; `trace_call` is the trace scope. - -```python -from digintlab_langfuse import ObservationKind - -with telemetry.trace_call( - name="ingest", - session_id="s-1", - user_id="u-1", - root_observation=True, - root_kind=ObservationKind.CHAIN, - tags=["batch"], -): - ... -``` - -### `spans` - -Observation helpers over context-local active-parent state: `observe_node_span`, `observe_generation`, `emit_guardrail_observation`, and `current_trace_id`. `trace_runtime_context` returns a `TraceRuntimeContext` for work that crosses a process or queue boundary. `scrub_auth_header` and `scrub_auth_header_async` are the HTTPX hooks behind `redact_http_auth_headers`. All helpers open observations only inside an active trace. - -```python -from digintlab_langfuse.spans import ObservationKind, observe_generation, observe_node_span - -with observe_node_span("retrieve", enabled=True, kind=ObservationKind.RETRIEVER, client_getter=lambda: telemetry.client): - with observe_generation("answer", model="my-model", client_getter=lambda: telemetry.client) as generation: - if generation is not None: - generation.update(output="done") -``` - -### `callbacks` - -`ToolSpanCallback` is a LangChain callback handler that records each tool call as a Langfuse tool observation under the active node span. `summarize_output` returns a `ToolObservationResult` (`output`, `level`, `status_message`) to control what is stored. The tool input is always captured. - -```python -from digintlab_langfuse.callbacks import ToolObservationResult - - -def summarize(output: object) -> ToolObservationResult: - return {"output": {"status": "ok", "chars": len(str(output))}} - - -callback = telemetry.tool_span_callback(summarize_output=summarize) -``` - -### `litellm` - -`with_litellm_proxy_trace_metadata` merges the active trace ids into `extra_body.metadata` of LiteLLM proxy kwargs, and returns the kwargs unchanged when no trace is active. Pass `trace_context=` to use a captured `TraceRuntimeContext` instead of the active trace. It raises `TypeError` if `extra_body` is not a dict. - -```python -from digintlab_langfuse.litellm import with_litellm_proxy_trace_metadata - -kwargs = with_litellm_proxy_trace_metadata({"model": "my-model"}, trace_context=telemetry.trace_runtime_context()) -``` - -### `scores` - -One typed API for all score kinds: `NumericScore`, `BooleanScore`, `CategoricalScore`, `TextScore`, and `CorrectionScore`. `ScoreTarget` selects the trace, observation, or session (`run_id`, `trace_id`, `observation_id`, `session_id`, `dataset_run_id`, `score_id`, `comment`, `config_id`, `metadata`). `score_elapsed` measures a block and emits its seconds as a numeric score. -```python -from digintlab_langfuse.scores import BooleanScore, ScoreTarget, emit_score, score_elapsed - -emit_score(telemetry.client, score=BooleanScore(name="passed", value=True), target=ScoreTarget(trace_id=trace_id)) -with score_elapsed(telemetry.client, name="latency_seconds", target=ScoreTarget(trace_id=trace_id), digits=3): - ... -``` - -### `prompts` - -Typed requests for fetching and creating managed prompts. `TextPromptRequest` and `ChatPromptRequest` take `name`, `version`, `label`, `cache_ttl_seconds`, `fallback`, `max_retries`, and `fetch_timeout_seconds`. `TextManagedPrompt` and `ChatManagedPrompt` take `name`, `prompt`, `labels`, `tags`, `config`, and `commit_message`. `create_or_update_prompt` creates a new prompt version. - -```python -from digintlab_langfuse.prompts import TextManagedPrompt, TextPromptRequest, create_or_update_prompt, get_prompt - -create_or_update_prompt(telemetry.client, TextManagedPrompt(name="greeting", prompt="Hello {{name}}", labels=("production",))) -prompt = get_prompt(telemetry.client, TextPromptRequest(name="greeting", label="production")) -``` - -### `experiments` - -Thin helpers over Langfuse datasets and local experiments. `run_local_experiment` normalizes items to JSON values and takes `evaluators`, `run_evaluators`, `max_concurrency` (default 50), `metadata`, and `experiment_id`. - -```python -from digintlab_langfuse.experiments import DatasetConfig, create_dataset, run_local_experiment - -create_dataset(telemetry.client, DatasetConfig(name="smoke")) -result = run_local_experiment( - telemetry.client, - name="smoke-run", - data=[{"input": "2+2", "expected_output": "4"}], - task=lambda *, item, **kwargs: "4", -) -``` - -`create_dataset_item` and the `ExperimentItem` type are also exported. - -### `budgets` - -Numeric ceiling checks for CI-style gates. `check_numeric_budgets` returns a `NumericBudgetViolation` for each missing or exceeded measurement. `validate_numeric_budget_ceilings` validates raw ceilings (optionally against `allowed_names`) and rejects non-numeric or unknown entries. - -```python -from digintlab_langfuse.budgets import check_numeric_budgets, validate_numeric_budget_ceilings - -ceilings = validate_numeric_budget_ceilings({"p95_seconds": 2.0}) -for violation in check_numeric_budgets({"p95_seconds": 3.1}, ceilings): - print(violation.name, violation.message) # p95_seconds 3.1 > 2.0 -``` - -### `timing` - -`elapsed_timer` times a local block. `create_span_timing_recorder` adds a `SpanTimingProcessor` to a tracer provider (a new one by default) and aggregates ended spans into `SpanTimingBucket` values keyed by `default_span_timing_key` or your own `key`. Beyond `max_buckets` (default 1000), new keys fold into `__overflow__`. Pass `recorder.tracer_provider` as `LangfuseClientConfig.tracer_provider` to record Langfuse spans. - -```python -from digintlab_langfuse.timing import create_span_timing_recorder, elapsed_timer - -with elapsed_timer() as timer: - ... -print(timer.rounded(3)) - -recorder = create_span_timing_recorder() -with recorder.tracer_provider.get_tracer("demo").start_as_current_span("work"): - ... -buckets = recorder.snapshot(reset=True) + telemetry.emit_score(score=NumericScore(name="quality", value=0.95), target=ScoreTarget(trace_id=trace_id)) ``` -### `redaction` - -Opt-in masking built from a `SensitiveDataRedactionConfig` (`replacement="[REDACTED]"`, `sensitive_field_names`, `pii_entities`, `pii_score_threshold=0.4`, `redact_bearer_tokens=True`). `create_sensitive_data_mask` and `create_sensitive_data_otel_mask` build the two Langfuse hooks; `redact_sensitive_data` applies the policy to any value directly. See [Sensitive data](#sensitive-data). +Create one telemetry owner per process, after workers fork, and don't construct `Langfuse(...)` yourself. Leaving the `async with` block flushes and shuts down; `await telemetry.flush()` exports without closing. -```python -from digintlab_langfuse import SensitiveDataRedactionConfig -from digintlab_langfuse.redaction import redact_sensitive_data - -policy = SensitiveDataRedactionConfig(pii_entities=frozenset({"EMAIL_ADDRESS"})) -redact_sensitive_data({"password": "hunter2", "note": "mail a@example.com"}, policy) -# {"password": "[REDACTED]", "note": "mail [REDACTED]"} -``` +To continue a trace in another process, capture `telemetry.trace_runtime_context()` and pass it to `with_litellm_proxy_trace_metadata()` on the other side. -### `json_values` - -`JsonValue` is the recursive JSON type used across the API. `to_json_value` converts enums, mappings, sequences, and other objects (via `str`) into it, and raises `ValueError` for non-finite floats. +## Modules -```python -from digintlab_langfuse.json_values import JsonValue, to_json_value +- `client`: `LangfuseClientConfig`, `init_langfuse_telemetry`, and `LangfuseTelemetry` (`trace_call`, `flush`, `shutdown`, and wrappers for the helpers below). +- `spans`: node, generation, and guardrail observations; the HTTPX auth-header scrubbing hooks. +- `callbacks`: `ToolSpanCallback`, a LangChain handler that records tool calls. +- `litellm`: adds the active trace ids to LiteLLM proxy request metadata. +- `scores`: one typed score API (`NumericScore`, `BooleanScore`, `CategoricalScore`, `TextScore`, `CorrectionScore`). +- `prompts`, `experiments`: thin typed wrappers for Langfuse prompts, datasets, and local experiments. +- `budgets`, `timing`: numeric ceiling checks and span timing aggregation. +- `redaction`, `json_values`: the redaction policy and the JSON value type. -payload: JsonValue = to_json_value({"count": 3, "tags": ("a", "b")}) -``` +For all configuration fields and their defaults, see `LangfuseClientConfig` in [`client.py`](src/digintlab_langfuse/client.py). ## Sensitive data -PII redaction is optional: - -```bash -uv add "digintlab-langfuse-python[redaction] @ git+https://github.com/DigintLab/digintlab-langfuse-python@v0.4.0" -``` - -With pip, pass the same requirement string to `pip install`. +Nothing is redacted by default. To mask credential fields, bearer tokens, and PII before export: ```python from digintlab_langfuse import LangfuseClientConfig, SensitiveDataRedactionConfig @@ -271,50 +56,24 @@ from digintlab_langfuse import LangfuseClientConfig, SensitiveDataRedactionConfi config = LangfuseClientConfig( public_key=public_key, secret_key=secret_key, - sensitive_data_redaction=SensitiveDataRedactionConfig( - pii_entities=frozenset({"EMAIL_ADDRESS", "IP_ADDRESS", "PHONE_NUMBER"}), - pii_score_threshold=0.4, - redact_bearer_tokens=True, - ), + sensitive_data_redaction=SensitiveDataRedactionConfig(), ) ``` -The policy masks common credential fields and bearer tokens, then uses Presidio's local rule-based recognizers for PII. It uses Presidio's bundled domain-suffix snapshot and does not load an NLP model, call an LLM, or make network requests. Configure `pii_entities` and `pii_score_threshold` for the data you expect; use `mask` and `mask_otel_spans` for a fully custom policy. +PII detection uses Presidio's local rule-based recognizers. It loads no NLP model and makes no network calls, so it won't catch names, addresses, or non-English text. For a custom policy, pass `mask` and `mask_otel_spans` instead. -> [!WARNING] -> Do not put sensitive data in span names, identifiers, links, events, or resource attributes. OpenTelemetry export hooks cannot rewrite them. -> -> `SensitiveDataRedactionConfig` also does not sanitize metadata forwarded to LiteLLM. Keep runtime-context, run-manager, and request metadata non-sensitive, -> or sanitize it before calling `with_litellm_proxy_trace_metadata()`. +To strip `Authorization` headers from HTTPX spans, set `redact_http_auth_headers=True` and initialize with `instrument_httpx=True`. -HTTP authorization-header redaction is separate. Enable both `instrument_httpx=True` and `redact_http_auth_headers=True`. Tool callbacks always capture the tool input and capture the output unless you provide `summarize_output`; enable masking to cover both. +> [!WARNING] +> Span names, identifiers, links, events, resource attributes, and metadata forwarded to LiteLLM are never rewritten. Keep secrets out of them. ## Limitations -- **One Langfuse project per process.** A second `init_langfuse_telemetry()` raises `RuntimeError`, whether it reuses the public key or uses another project's. It also raises when a Langfuse SDK resource was created outside this package. `shutdown()` does not release this ownership, so initialize once per process lifetime. Run separate projects in separate processes. -- **Langfuse is limited to one minor version** (`langfuse>=5.2.0,<5.3`). The package reaches into SDK internals, which `tests/package_contract/test_langfuse_internals.py` checks on every dependency update, so a new Langfuse minor release is only allowed after review. Do not override the range. -- **Private SDK internals are used.** Ownership checks and lifecycle handling read `LangfuseResourceManager` state (`_instances`, `_lock`) and the client's `_resources`. `ToolSpanCallback` is typed against `LangfuseObservationWrapper` from `langfuse._client.span`. -- **Observation helpers fail open and no-op quietly.** Errors in `trace_call`, `observe_node_span`, `observe_generation`, `emit_guardrail_observation`, `ToolSpanCallback`, and `with_litellm_proxy_trace_metadata` are logged and never replace your result or exception. Outside an active trace these helpers record nothing. Configuration, scoring, `flush`, and `shutdown` fail fast. -- **Instrumentation is process-wide.** HTTPX and botocore instrumentors are shared by every owner in the process and must use the same tracer provider and redaction setting, or initialization raises. Instrumentation installed by other code is left in place, and HTTPX initialization fails if that code owns it and `redact_http_auth_headers=True`. `shutdown()` uninstruments only what this package installed. -- **Redaction has limits.** - - It runs through Langfuse's `mask` and `mask_otel_spans` hooks, so it covers masked data and string span attributes only. Span names, identifiers, links, events, and resource attributes are not rewritten. - - Metadata forwarded to LiteLLM is not sanitized. - - Matching is by field name (normalized, so `apiKey` matches `api_key`), bearer-token pattern, and Presidio's English rule-based recognizers. There is no NLP model, so names and addresses are not detected, and non-English text is not analyzed. - - Secrets in free text without a recognized field name or bearer prefix are not masked. - - `redact_http_auth_headers` only masks `Authorization`; other headers such as cookies or API-key headers are not scrubbed. - - The default `pii_entities` apply when you enable `sensitive_data_redaction`; pass an empty set to skip PII analysis and the `redaction` extra. - -## Examples - -Runnable scripts live in [`examples/`](examples). Each reads `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, and `LANGFUSE_BASE_URL`, and exits with a message if one is unset: - -- [`basic_tracing.py`](examples/basic_tracing.py): trace one call and attach numeric, categorical, and elapsed-time scores. -- [`langchain_tool_callback.py`](examples/langchain_tool_callback.py): record a LangChain tool call as a tool observation. -- [`redaction.py`](examples/redaction.py): enable the built-in redaction policy. - -```bash -uv run python examples/basic_tracing.py -``` +- One Langfuse project per process. A second `init_langfuse_telemetry()` raises, and `shutdown()` does not release the claim. +- The package uses private Langfuse SDK internals, so Langfuse is limited to one minor version (`>=5.2.0,<5.3`). `tests/package_contract/test_langfuse_internals.py` catches breaking changes. +- Observation helpers fail open: errors are logged and never replace your result. Configuration, scoring, flush, and shutdown raise. +- HTTPX and botocore instrumentation is opt-in and process-wide. Instrumentation owned by other code is left alone. +- Plain `http://` endpoints need `allow_insecure_http=True`. ## Development @@ -325,24 +84,17 @@ task check task test ``` -Run the authenticated E2E suite only against a disposable Langfuse project: +The E2E suite needs a disposable Langfuse project: ```bash -RUN_LANGFUSE_E2E=1 \ -LANGFUSE_E2E_PUBLIC_KEY=... \ -LANGFUSE_E2E_SECRET_KEY=... \ -LANGFUSE_E2E_BASE_URL=https://cloud.langfuse.com \ -task test-e2e +RUN_LANGFUSE_E2E=1 LANGFUSE_E2E_PUBLIC_KEY=... LANGFUSE_E2E_SECRET_KEY=... \ +LANGFUSE_E2E_BASE_URL=https://cloud.langfuse.com task test-e2e ``` -## API stability - -The public API is the names listed in each public module's `__all__` plus the top-level `digintlab_langfuse` exports. Modules and names that start with an underscore are private. - -Versioning follows [SemVer](https://semver.org/). Before 1.0, a minor version (0.x to 0.(x+1)) may contain breaking changes, which are listed in `CHANGELOG.md` under "Changed" or "Removed". Patch versions are backward compatible. +## Versioning -Langfuse is limited to the patch releases of one tested minor version. Moving to a new Langfuse minor version is released as at least a minor version of this package. +The public API is each public module's `__all__` plus the top-level exports; anything starting with an underscore is private. Releases follow [SemVer](https://semver.org/). Before 1.0, minor releases may break things, and those changes are listed in `CHANGELOG.md`. Moving to a new Langfuse minor version is always at least a minor release. ## License -This package is licensed under the GNU Lesser General Public License v3.0 or later (`LGPL-3.0-or-later`). You may use it in any application, including proprietary applications that import the library unmodified. If you distribute a modified version of the library itself, those modifications must be made available under the LGPL. See [`COPYING.LESSER`](COPYING.LESSER) for the LGPL terms and [`COPYING`](COPYING) for the GNU GPL v3 text it builds on. +[LGPL-3.0-or-later](COPYING.LESSER). You can use the library in any application, proprietary ones included. If you distribute a modified copy of the library itself, you must share those changes under the LGPL. diff --git a/Taskfile.yml b/Taskfile.yml index 9081618..9bb9795 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -28,12 +28,12 @@ tasks: lint: desc: Run lint checks cmds: - - uv run ruff check src/ tests/ examples/ + - uv run ruff check src/ tests/ type-check: desc: Run static type checks cmds: - - uv run mypy src/ tests/ examples/ + - uv run mypy src/ tests/ check: desc: Run formatting, lint, and type checks diff --git a/examples/__init__.py b/examples/__init__.py deleted file mode 100644 index 1f7a088..0000000 --- a/examples/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Runnable usage examples for digintlab-langfuse-python.""" diff --git a/examples/basic_tracing.py b/examples/basic_tracing.py deleted file mode 100644 index 6fc0455..0000000 --- a/examples/basic_tracing.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Trace one call and attach scores. - -Run with LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY, and LANGFUSE_BASE_URL set. -""" - -import asyncio -import os -import sys - -from digintlab_langfuse import LangfuseClientConfig, init_langfuse_telemetry -from digintlab_langfuse.scores import CategoricalScore, NumericScore, ScoreTarget, score_elapsed - - -def config_from_env() -> LangfuseClientConfig: - """Build a client config from the environment, exiting with a clear message when it is incomplete.""" - names = ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_BASE_URL") - missing = [name for name in names if not os.environ.get(name)] - if missing: - sys.exit(f"Set these environment variables before running this example: {', '.join(missing)}") - return LangfuseClientConfig( - public_key=os.environ["LANGFUSE_PUBLIC_KEY"], - secret_key=os.environ["LANGFUSE_SECRET_KEY"], - base_url=os.environ["LANGFUSE_BASE_URL"], - ) - - -async def run(config: LangfuseClientConfig) -> None: - """Open a root observation, score the active trace, then emit scores against an explicit trace id.""" - telemetry = init_langfuse_telemetry(config) - async with telemetry: - with telemetry.trace_call(name="example-run", session_id="example-session", user_id="example-user", root_observation=True): - telemetry.score_current_trace(score=NumericScore(name="quality", value=0.95)) - runtime_context = telemetry.trace_runtime_context() - if runtime_context is not None: - target = ScoreTarget(trace_id=runtime_context.trace_id) - with score_elapsed(telemetry.client, name="work_seconds", target=target): - await asyncio.sleep(0.01) - telemetry.emit_score(score=CategoricalScore(name="outcome", value="ok"), target=target) - await telemetry.flush() - - -if __name__ == "__main__": - asyncio.run(run(config_from_env())) diff --git a/examples/langchain_tool_callback.py b/examples/langchain_tool_callback.py deleted file mode 100644 index 4546b4c..0000000 --- a/examples/langchain_tool_callback.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Record a LangChain tool call as a Langfuse tool observation. - -Run with LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY, and LANGFUSE_BASE_URL set. -""" - -import asyncio -import os -import sys - -from langchain_core.tools import tool - -from digintlab_langfuse import LangfuseClientConfig, ObservationKind, init_langfuse_telemetry - - -@tool -def add(left: int, right: int) -> int: - """Add two integers.""" - return left + right - - -def config_from_env() -> LangfuseClientConfig: - """Build a client config from the environment, exiting with a clear message when it is incomplete.""" - names = ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_BASE_URL") - missing = [name for name in names if not os.environ.get(name)] - if missing: - sys.exit(f"Set these environment variables before running this example: {', '.join(missing)}") - return LangfuseClientConfig( - public_key=os.environ["LANGFUSE_PUBLIC_KEY"], - secret_key=os.environ["LANGFUSE_SECRET_KEY"], - base_url=os.environ["LANGFUSE_BASE_URL"], - ) - - -async def run(config: LangfuseClientConfig) -> int: - """Invoke a tool inside a node observation so the callback has an active parent to attach to.""" - telemetry = init_langfuse_telemetry(config) - async with telemetry: - with ( - telemetry.trace_call(name="example-tools", session_id="example-session", user_id="example-user", root_observation=True), - telemetry.observe_node_span("tools-node", enabled=True, kind=ObservationKind.CHAIN), - ): - result: int = add.invoke({"left": 2, "right": 3}, config={"callbacks": [telemetry.tool_span_callback()]}) - await telemetry.flush() - return result - - -if __name__ == "__main__": - sys.stdout.write(f"add(2, 3) = {asyncio.run(run(config_from_env()))}\n") diff --git a/examples/redaction.py b/examples/redaction.py deleted file mode 100644 index 792f684..0000000 --- a/examples/redaction.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Mask credentials and PII before data leaves the process. - -Requires the `redaction` extra. Run with LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY, and LANGFUSE_BASE_URL set. -""" - -import asyncio -import os -import sys - -from digintlab_langfuse import LangfuseClientConfig, SensitiveDataRedactionConfig, init_langfuse_telemetry -from digintlab_langfuse.json_values import JsonValue -from digintlab_langfuse.redaction import redact_sensitive_data - - -def redaction_policy() -> SensitiveDataRedactionConfig: - """Return the example policy: default credential fields plus email and phone PII.""" - return SensitiveDataRedactionConfig(pii_entities=frozenset({"EMAIL_ADDRESS", "PHONE_NUMBER"})) - - -def config_from_env() -> LangfuseClientConfig: - """Build a client config with redaction enabled, exiting with a clear message when the environment is incomplete.""" - names = ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_BASE_URL") - missing = [name for name in names if not os.environ.get(name)] - if missing: - sys.exit(f"Set these environment variables before running this example: {', '.join(missing)}") - return LangfuseClientConfig( - public_key=os.environ["LANGFUSE_PUBLIC_KEY"], - secret_key=os.environ["LANGFUSE_SECRET_KEY"], - base_url=os.environ["LANGFUSE_BASE_URL"], - sensitive_data_redaction=redaction_policy(), - ) - - -async def run(config: LangfuseClientConfig) -> JsonValue: - """Trace a call whose input contains sensitive values, and return what the policy does to the same payload.""" - payload: dict[str, JsonValue] = {"api_key": "sk-secret", "note": "Contact jane.doe@example.com"} - telemetry = init_langfuse_telemetry(config) - async with telemetry: - with telemetry.trace_call( - name="example-redaction", - session_id="example-session", - user_id="example-user", - root_observation=True, - root_input=payload, - ): - pass - await telemetry.flush() - return redact_sensitive_data(payload, redaction_policy()) - - -if __name__ == "__main__": - sys.stdout.write(f"{asyncio.run(run(config_from_env()))}\n") diff --git a/tests/package_contract/test_examples.py b/tests/package_contract/test_examples.py deleted file mode 100644 index 7da311a..0000000 --- a/tests/package_contract/test_examples.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Offline checks that keep the scripts in `examples/` importable and runnable.""" - -import importlib.util -import py_compile -from pathlib import Path -from types import ModuleType -from uuid import uuid4 - -import httpx -import pytest -from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter - -from digintlab_langfuse import LangfuseClientConfig, SensitiveDataRedactionConfig - -EXAMPLES_DIR = Path(__file__).resolve().parents[2] / "examples" -EXAMPLE_PATHS = sorted(path for path in EXAMPLES_DIR.glob("*.py") if path.name != "__init__.py") -ENV_NAMES = ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_BASE_URL") - - -def _load(path: Path) -> ModuleType: - spec = importlib.util.spec_from_file_location(f"_example_{path.stem}", path) - assert spec is not None - assert spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def _offline_config(http_client: httpx.Client, sensitive_data_redaction: SensitiveDataRedactionConfig | None = None) -> LangfuseClientConfig: - return LangfuseClientConfig( - public_key=f"pk-example-{uuid4()}", - secret_key="test-secret", - httpx_client=http_client, - span_exporter=InMemorySpanExporter(), - flush_at=100, - flush_interval=60.0, - sensitive_data_redaction=sensitive_data_redaction, - ) - - -def test_examples_exist() -> None: - assert {path.stem for path in EXAMPLE_PATHS} == {"basic_tracing", "langchain_tool_callback", "redaction"} - - -@pytest.mark.parametrize("path", EXAMPLE_PATHS, ids=lambda path: path.stem) -def test_example_compiles_and_exposes_entry_points(path: Path, tmp_path: Path) -> None: - py_compile.compile(str(path), cfile=str(tmp_path / f"{path.stem}.pyc"), doraise=True) - module = _load(path) - assert callable(module.run) - assert callable(module.config_from_env) - - -@pytest.mark.parametrize("path", EXAMPLE_PATHS, ids=lambda path: path.stem) -def test_example_exits_with_clear_message_without_credentials(path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - for name in ENV_NAMES: - monkeypatch.delenv(name, raising=False) - module = _load(path) - with pytest.raises(SystemExit) as exit_info: - module.config_from_env() - message = str(exit_info.value) - assert all(name in message for name in ENV_NAMES) - - -@pytest.mark.parametrize("path", EXAMPLE_PATHS, ids=lambda path: path.stem) -def test_example_reads_credentials_from_environment(path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-env") - monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-env") - monkeypatch.setenv("LANGFUSE_BASE_URL", "https://langfuse.example.com") - config = _load(path).config_from_env() - assert isinstance(config, LangfuseClientConfig) - assert config.public_key == "pk-env" - assert config.base_url == "https://langfuse.example.com" - - -# The pinned Langfuse SDK calls archived backoff 2.2.1 from its score worker, which uses a deprecated asyncio helper on Python 3.14+. -@pytest.mark.filterwarnings("ignore:'asyncio.iscoroutinefunction' is deprecated:DeprecationWarning:backoff._decorator") -@pytest.mark.asyncio -async def test_basic_tracing_example_runs_offline() -> None: - module = _load(EXAMPLES_DIR / "basic_tracing.py") - with httpx.Client(transport=httpx.MockTransport(lambda _request: httpx.Response(202, json={"message": "Accepted"}))) as http_client: - await module.run(_offline_config(http_client)) - - -@pytest.mark.asyncio -async def test_langchain_tool_callback_example_runs_offline() -> None: - module = _load(EXAMPLES_DIR / "langchain_tool_callback.py") - with httpx.Client(transport=httpx.MockTransport(lambda _request: httpx.Response(202, json={"message": "Accepted"}))) as http_client: - assert await module.run(_offline_config(http_client)) == 5 - - -@pytest.mark.asyncio -async def test_redaction_example_runs_offline() -> None: - module = _load(EXAMPLES_DIR / "redaction.py") - with httpx.Client(transport=httpx.MockTransport(lambda _request: httpx.Response(202, json={"message": "Accepted"}))) as http_client: - redacted = await module.run(_offline_config(http_client, module.redaction_policy())) - assert redacted == {"api_key": "[REDACTED]", "note": "Contact [REDACTED]"}