From c6e9426e43f4ff0a6a1e7e5ef14647b76d0d6be6 Mon Sep 17 00:00:00 2001 From: saileshwar-skyflow Date: Mon, 9 Nov 2026 14:11:05 +0530 Subject: [PATCH] SK-3289: address Detect docs and sample parity feedback (Python) Fix the Detect README and sample gaps raised in the final Detect feedback, for both the flowvault and skyvault Python SDKs. flowvault README: - drop the unsupported SKYFLOW_ID file source (the enum is BASE64/PRESIGNED_URL only) - document Entity/CustomType include_prefix and prefix_override, and CustomType deidentification_type and destination - add where-to-create pointers for the configuration id, token groups and redaction patterns - add response examples and a common-errors section - make the re-identify example a round trip; branch re-identify file on output_type skyvault README: - make the re-identify example a round trip that stores tokens in a table.column; branch re-identify file on output_type - add a bold warning that deidentify_file no longer waits by default - note the server's default toxicity check; fix the stale v1 EOL banner samples (both SDKs): print request_id in Detect error handlers. Nomenclature: drop the "Detect V2" label from both READMEs and the samples README; rename the migration guide to docs/migrate_detect.md, retitle it, cite skyflow 3.0.0, and promote the deidentify_file behaviour change. setup.py version is managed by release automation and left unchanged; the guide now names 3.0.0 as the target. Co-Authored-By: Claude Opus 4.8 --- ...e_detect_v1_to_v2.md => migrate_detect.md} | 36 ++++--- flowvault/README.md | 100 +++++++++++++++--- flowvault/samples/README.md | 2 +- .../samples/detect_api/check_guardrails.py | 1 + .../samples/detect_api/deidentify_file.py | 3 + .../samples/detect_api/deidentify_string.py | 2 + flowvault/samples/detect_api/get_run.py | 1 + .../samples/detect_api/reidentify_file.py | 1 + .../samples/detect_api/reidentify_string.py | 1 + skyvault/README.md | 46 ++++++-- .../samples/detect_api/check_guardrails.py | 1 + .../samples/detect_api/deidentify_file.py | 1 + .../samples/detect_api/deidentify_string.py | 2 + skyvault/samples/detect_api/get_run.py | 1 + .../samples/detect_api/reidentify_file.py | 1 + .../samples/detect_api/reidentify_string.py | 1 + 16 files changed, 159 insertions(+), 41 deletions(-) rename docs/{migrate_detect_v1_to_v2.md => migrate_detect.md} (91%) diff --git a/docs/migrate_detect_v1_to_v2.md b/docs/migrate_detect.md similarity index 91% rename from docs/migrate_detect_v1_to_v2.md rename to docs/migrate_detect.md index 28e39524..12b3d45e 100644 --- a/docs/migrate_detect_v1_to_v2.md +++ b/docs/migrate_detect.md @@ -1,14 +1,22 @@ -# Migrate Detect from V1 to V2 (skyvault Python SDK) - -The skyvault Python SDK replaces the Detect V1 methods with the V2 methods, using the -same method and class names as the flowvault Detect SDK. There is no `V2` suffix -anywhere. `deidentify_file` keeps its name and takes the V2 `DeidentifyFileRequest`, -returning the V2 `DeidentifyFileResponse`. Because the V1 methods and classes are -removed, there is no name conflict. All V2 classes live in `skyflow.vault.detect` and -the enums in `skyflow.utils.enums`. - -V1 callers must migrate when they upgrade, so this ships as a major release. Client -creation and the `skyflow_client.detect()` accessor do not change. +# Migrate to the new Detect methods (skyflow 3.0.0) + +The skyvault Python SDK replaces the previous Detect methods with the new Detect methods, +using the same method and class names as the flowvault Detect SDK — there is no version +suffix on any of them. `deidentify_file` keeps its name and now takes the new +`DeidentifyFileRequest`, returning the new `DeidentifyFileResponse`. Because the previous +methods and classes are removed, there is no name conflict. All Detect classes live in +`skyflow.vault.detect` and the enums in `skyflow.utils.enums`. + +Callers of the previous Detect methods must migrate when they upgrade, so this ships in the +next major release, **skyflow 3.0.0** — pin or upgrade accordingly (`pip install "skyflow==3.0.0"`). +Client creation and the `skyflow_client.detect()` accessor do not change. + +> **⚠️ Biggest behaviour change:** `deidentify_file` **no longer waits for the result by +> default.** It used to always poll; now, without `poll_options`, it submits and returns +> immediately — `status` is `None`, `output` is empty, and nothing is written to disk. Pass +> `poll_options` to block until the run finishes, or retrieve the result later with `get_run`. +> Code ported unchanged runs without error but produces no outputs, so the failure surfaces +> downstream rather than at the call site. ## What changes @@ -355,7 +363,7 @@ fields as the V2 `DeidentifyFileResponse` (`run_id`, `status`, `output_type`, `o ## Timeline -| Release | V1 methods | V2 methods | +| Release | Previous Detect methods | New Detect methods | | --- | --- | --- | -| skyvault 2.1.x | available | not available | -| next major | removed | available under the flowvault names | +| skyflow 2.1.x | available | not available | +| skyflow 3.0.0 | removed | available under the flowvault names | diff --git a/flowvault/README.md b/flowvault/README.md index 3e1d2773..faf17b12 100644 --- a/flowvault/README.md +++ b/flowvault/README.md @@ -983,7 +983,7 @@ The same pattern applies to every operation, via its corresponding options class # Detect -Skyflow Detect finds and de-identifies sensitive data in strings and files, re-identifies tokens on demand, and screens text against guardrails. The flowvault SDK exposes the Detect V2 API through `DetectController`, returned by `skyflow_client.detect()` and `skyflow_client.detect('')`. Every request is scoped to that vault, and the controller shares the same credentials, timeouts and retries as `vault()`; the `vault_id` is filled in from the vault config for every method. +Skyflow Detect finds and de-identifies sensitive data in strings and files, re-identifies tokens on demand, and screens text against guardrails. The flowvault SDK exposes the Detect API through `DetectController`, returned by `skyflow_client.detect()` and `skyflow_client.detect('')`. Every request is scoped to that vault, and the controller shares the same credentials, timeouts and retries as `vault()`; the `vault_id` is filled in from the vault config for every method. Detect configurations are created in Studio or through the management API — the SDK does not create, update or delete them. Import request/response models from `skyflow.vault.detect` and enums from `skyflow.utils.enums`. @@ -1003,8 +1003,8 @@ detect = skyflow_client.detect() ## Before you start - **Role.** The service account needs the Detect Invoker, Detect Manager or Vault Owner role on the vault. Detect Invoker can de-identify data and write vault tokens into the vault; Detect Manager adds the ability to create, update and delete Detect configurations. Re-identifying values stored as `VAULT_TOKEN` is governed like detokenization: the caller also needs a policy granting the Detokenization action on the destination, unless it is a Vault Owner. -- **Configuration.** Pass `configuration_id` (the id of a stored Detect configuration) or an inline `configuration` (a `DetectConfiguration`). Both are optional: with neither set the SDK sends only the vault id and the run uses the vault's default Detect settings; sending both is rejected by the API with HTTP 400. -- **Destination for vault tokens.** An entity de-identified with `VAULT_TOKEN` stores its value in the token group named by `destination`. That token group must already exist in the vault; Detect writes into it but does not create it. +- **Configuration.** Pass `configuration_id` (the id of a stored Detect configuration) or an inline `configuration` (a `DetectConfiguration`). Both are optional: with neither set the SDK sends only the vault id and the run uses the vault's default Detect settings; sending both is rejected by the API with HTTP 400. Create and manage stored configurations — and copy their ids — in Studio (vault → **Detect**) or through the management API; the SDK does not create them. +- **Destination for vault tokens.** An entity de-identified with `VAULT_TOKEN` stores its value in the token group named by `destination`. That token group must already exist in the vault; Detect writes into it but does not create it. Create token groups, and the named redaction patterns that re-identification applies, in the vault schema in Studio. - **API reference.** Every Detect request, response and configuration class is listed in the [API Reference](../docs/api_reference.md). ## deidentify_string @@ -1060,9 +1060,9 @@ Returns a `DeidentifyStringResponse` with `processed_text`, `entities` (each a ` | Class | Fields | Notes | |-------|--------|-------| | `Detect` | `entities`, `object_entities`, `restrict`, `skip`, `custom_types`, `return_entities` | `restrict` / `skip` are regexes always / never treated as sensitive. `return_entities` is `EXCLUDE_SENSITIVE_DATA` (server default), `ALL` or `NONE`. | -| `Entity` | `entity_type`, `deidentification_type`, `destination`, `transformation` | `entity_type` defaults to `ALL`. `deidentification_type` is `ENTITY_UNIQUE_COUNTER` (server default), `ENTITY_ONLY`, `MASK` or `VAULT_TOKEN`. `destination` (a token group name) is used only with `VAULT_TOKEN`. | +| `Entity` | `entity_type`, `deidentification_type`, `destination`, `transformation`, `include_prefix`, `prefix_override` | `entity_type` defaults to `ALL`. `deidentification_type` is `ENTITY_UNIQUE_COUNTER` (server default), `ENTITY_ONLY`, `MASK` or `VAULT_TOKEN`. `destination` (a token group name) is used only with `VAULT_TOKEN`. `include_prefix` (`IncludePrefix.YES` / `IncludePrefix.NO`) and `prefix_override` also apply only to `VAULT_TOKEN`: `include_prefix=IncludePrefix.NO` stores a raw token that **cannot be re-identified** (a data-loss trap), and `prefix_override` replaces the entity-type label in the token with a custom one. | | `Transformation` / `ShiftDates` | `shift_dates`; `min_days`, `max_days` | Shifts detected dates by a random number of days within the range. | -| `CustomType` | `label`, `match` | A regex-defined entity type; `label` is reported as the entity type of its matches. | +| `CustomType` | `label`, `match`, `deidentification_type`, `destination`, `include_prefix`, `prefix_override` | A regex-defined entity type; `label` is reported as the entity type of its matches. `deidentification_type`, `destination`, `include_prefix` and `prefix_override` behave exactly as on `Entity`. | | `ObjectEntity` | `entity_type`, `deidentification_type` | Visual objects in images: `FACE`, `LICENSE_PLATE`, `LOGO`, `SIGNATURE` or `ALL`, handled with `REDACT` or `UNREDACT`. File inputs only. | | `FileMapping` | `source`, `destination`, `output_file_name_suffix`, `entities_destination`, `object_entities_destination` | Where the processed file and entity files are written. File inputs only. | | `Media` | `audio`, `document`, `image` | Media-type options. File inputs only. | @@ -1108,7 +1108,7 @@ configuration = DetectConfiguration( De-identifies a file. The API is asynchronous: the submit call returns a run id and the result is retrieved from the run. Build the request with `DeidentifyFileRequest`. -Supply the file in exactly one of three ways: `file_path`, `file`, or `value` plus a `DeidentifySourceType` `data_source` (`BASE64`, the `SKYFLOW_ID` of a file record in the vault, or `PRESIGNED_URL`). A local `file` / `file_path` is read when the request is sent, base64-encoded, and its `data_format` inferred from the extension. Passing more than one of them, or none, fails client-side validation before any network call. `configuration_id` / `configuration` follow the same rules as for strings (see [Before you start](#before-you-start)). +Supply the file in exactly one of three ways: `file_path`, `file`, or `value` plus a `DeidentifySourceType` `data_source` (`BASE64` or `PRESIGNED_URL`). flowvault does **not** support a Skyflow ID as a file source — its `DeidentifySourceType` has only `BASE64` and `PRESIGNED_URL`. A local `file` / `file_path` is read when the request is sent, base64-encoded, and its `data_format` inferred from the extension. Passing more than one of them, or none, fails client-side validation before any network call. `configuration_id` / `configuration` follow the same rules as for strings (see [Before you start](#before-you-start)). - **Submit only** (default): one call, returns `run_id`. `status` is `None` and nothing is written to disk — retrieve the result later with [`get_run`](#get_run). - **Submit and poll**: pass `poll_options`. The SDK polls with exponential backoff (1, 2, 4, 8, 16, then 32 seconds) until the run is `SUCCESS`, `FAILED` or `UNKNOWN`, or until `wait_time` (default 60 s, max 300) or `max_attempts` (default and max 23) is used up. If the budget runs out the response carries the run id with `IN_PROGRESS` or `QUEUED`, so you can resume with `get_run`. `output_directory` is valid only with `poll_options`; setting it without them fails client-side validation. @@ -1149,7 +1149,7 @@ else: # IN_PROGRESS or QUEUED: budget ran out, resume - `/data/processed/processed-entities-patient-notes.json` — the detected entities - `/data/processed/processed-object-entities-patient-notes.json` — object entities, when the run produced them -`output_directory` is created if it does not exist. **If you poll a local file and omit `output_directory`, these files are written to the current working directory** — often your project root or a server's working directory — so set `output_directory` explicitly. A request built from a `SKYFLOW_ID` or `PRESIGNED_URL` value has no local name to derive from, and presigned-URL outputs are not fetched, so in those cases read `response.output` and write the files yourself. +`output_directory` is created if it does not exist. **If you poll a local file and omit `output_directory`, these files are written to the current working directory** — often your project root or a server's working directory — so set `output_directory` explicitly. A request built from a `PRESIGNED_URL` value has no local name to derive from, and presigned-URL outputs are not fetched, so in that case read `response.output` and write the files yourself. **Supported formats and limits.** `data_format` accepts every `FileDataFormat`: `bmp`, `csv`, `dcm`, `doc`, `docx`, `gif`, `jpeg`, `jpg`, `json`, `jsonl`, `mp3`, `pdf`, `png`, `ppt`, `pptx`, `tif`, `tiff`, `txt`, `wav`, `xls`, `xlsx`, `xml`, `zip`. The `VAULT_TOKEN` de-identification type is supported only for `txt`, `csv`, `json`, `jsonl`, `xml`, `docx`, `xlsx` and `pptx`. @@ -1200,18 +1200,37 @@ Synchronous. Re-identification looks tokens up in the vault, so it applies only Build the request with `ReidentifyStringRequest`: the tokenized text and an optional list of `RedactionLevel` rules. On flowvault vaults each rule targets a `token_group_name` and applies a named `redaction_pattern` configured in the vault. Without rules every token is replaced by its original value. +Only text that actually contains vault tokens can be re-identified, so the example below is a round trip: first de-identify with `VAULT_TOKEN` to mint tokens, then re-identify the `processed_text` that comes back. A vault token is an opaque value like `a1b2c3d4-...` (optionally prefixed, e.g. `[NAME_a1b2c3d4-...]`), not the `[NAME_1]` counter placeholder produced by `ENTITY_UNIQUE_COUNTER`. + ```python -from skyflow.vault.detect import ReidentifyStringRequest, RedactionLevel +from skyflow.vault.detect import ( + DeidentifyStringRequest, DetectConfiguration, Detect, Entity, + ReidentifyStringRequest, RedactionLevel, +) +from skyflow.utils.enums import EntityType, DeidentificationType + +# 1. De-identify with VAULT_TOKEN so the value is stored and can be re-identified later. +deidentified = skyflow_client.detect().deidentify_string( + DeidentifyStringRequest( + text='My name is John Doe.', + configuration=DetectConfiguration(detect=Detect(entities=[ + Entity(entity_type=EntityType.NAME, deidentification_type=DeidentificationType.VAULT_TOKEN, destination=''), + ])), + ) +) +# deidentified.processed_text now holds a vault token, e.g. 'My name is a1b2c3d4-...' +# 2. Re-identify those tokens. response = skyflow_client.detect().reidentify_string( ReidentifyStringRequest( - text='My name is [NAME_1].', + text=deidentified.processed_text, redaction_level=[ RedactionLevel(token_group_name='', redaction_pattern=''), ], ) ) print(response.processed_text) +# My name is John Doe. ``` > The flowvault `RedactionLevel` has only `token_group_name` and `redaction_pattern`; the skyvault SDK's has `entity_name` and `redaction_type` instead. Token groups and redaction patterns are configured in the vault. @@ -1227,6 +1246,7 @@ import base64 from skyflow.vault.detect import ReidentifyFileRequest, RedactionLevel from skyflow.utils.enums import ReidentifySourceType, FileDataFormat, DetectRunStatus +data_format = FileDataFormat.TXT with open('/data/redacted/notes.txt', 'rb') as handle: encoded = base64.b64encode(handle.read()).decode('utf-8') @@ -1234,23 +1254,26 @@ response = skyflow_client.detect().reidentify_file( ReidentifyFileRequest( data_source=ReidentifySourceType.BASE64, value=encoded, - data_format=FileDataFormat.TXT, + data_format=data_format, redaction_level=[RedactionLevel(token_group_name='', redaction_pattern='')], ) ) if response.status == DetectRunStatus.SUCCESS: for index, output in enumerate(response.output): - # output.processed_file is base64 or a presigned URL, per response.output_type - with open(f'reidentified-{index}.txt', 'wb') as out: - out.write(base64.b64decode(output.processed_file)) + # Branch on output_type: base64 bytes are decoded, a presigned URL is downloaded. + if response.output_type == ReidentifySourceType.BASE64: + with open(f'reidentified-{index}.{data_format.value}', 'wb') as out: + out.write(base64.b64decode(output.processed_file)) + else: # ReidentifySourceType.PRESIGNED_URL — processed_file is a URL to fetch + print('download re-identified file from', output.processed_file) ``` -Returns a `ReidentifyFileResponse` with `status`, `output_type`, `output` (a list of `FileOutput`) and `metrics`. +Returns a `ReidentifyFileResponse` with `status`, `output_type` (a `ReidentifySourceType`), `output` (a list of `FileOutput`) and `metrics`. ## check_guardrails -Synchronous. Screens text for toxicity and a list of denied topics — a natural pre-LLM check. Build the request with `CheckGuardrailsRequest`. `check_toxicity` and `deny_topics` are optional; the SDK sends each only when set and fills in no default. The response is passed through as the API returns it. +Synchronous. Screens text for toxicity and a list of denied topics — a natural pre-LLM check. Build the request with `CheckGuardrailsRequest`. `check_toxicity` and `deny_topics` are optional; the SDK sends each only when set and fills in no default. **When `check_toxicity` is omitted, the server still checks toxicity by default.** The response is passed through as the API returns it. ```python from skyflow.vault.detect import CheckGuardrailsRequest @@ -1268,6 +1291,53 @@ Returns a `CheckGuardrailsResponse` with `text`, `validation` (`GuardrailsValida All Detect methods raise `skyflow.error.SkyflowError` on validation failure, credential failure, or an API error — see [Error Handling](#error-handling). Client-side validation fails before any network call. +## Response examples and common errors + +Enum-typed fields (`status`, `output_type`, `validation`, …) are shown below by their string values. + +`deidentify_string`: + +```json +{ + "processed_text": "My name is [NAME_1], and my email is [EMAIL_ADDRESS_1].", + "entities": [ + { "token": "[NAME_1]", "value": "", "location": { "start_index": 11, "end_index": 19, "start_index_processed": 11, "end_index_processed": 19 }, "entity_type": "NAME", "entity_scores": { "NAME": 0.99 } } + ], + "metrics": { "word_count": 10, "character_count": 54 } +} +``` + +`deidentify_file` (polled) / `get_run`: + +```json +{ + "run_id": "run_a1b2c3d4", + "status": "SUCCESS", + "output_type": "BASE64", + "output": [ { "processed_file": "", "processed_file_type": "application/pdf", "processed_file_extension": "pdf" } ], + "metrics": { "pages": 3 }, + "message": null +} +``` + +`reidentify_string`: + +```json +{ "processed_text": "My name is John Doe.", "metrics": { "word_count": 5 } } +``` + +`check_guardrails`: + +```json +{ "text": "", "validation": "FAILED", "toxic": true, "denied_topic": false } +``` + +Common Detect errors (all raised as `SkyflowError`): + +- **HTTP 400 — both configuration sources set.** Passing `configuration_id` and `configuration` in the same request is rejected; pass at most one. +- **Missing token group.** A `VAULT_TOKEN` entity whose `destination` token group does not exist in the vault fails — create the token group first. +- **Unsupported format with `VAULT_TOKEN`.** `VAULT_TOKEN` is only valid for `txt`, `csv`, `json`, `jsonl`, `xml`, `docx`, `xlsx` and `pptx`; other `data_format` values are rejected. + # Error Handling ## Two layers of errors diff --git a/flowvault/samples/README.md b/flowvault/samples/README.md index 0737bc10..f5b4fc79 100644 --- a/flowvault/samples/README.md +++ b/flowvault/samples/README.md @@ -44,7 +44,7 @@ python flowvault/samples/vault_api/insert_records.py | [upload_files.py](vault_api/upload_files.py) | Upload files to file columns (`file_path` / `base64` / `file_object`) | | [delete_files.py](vault_api/delete_files.py) | Delete files from file columns (by `skyflow_id` or `unique_values`) | -## Detect V2 +## Detect | Sample | Demonstrates | |---|---| diff --git a/flowvault/samples/detect_api/check_guardrails.py b/flowvault/samples/detect_api/check_guardrails.py index 54082337..5e338281 100644 --- a/flowvault/samples/detect_api/check_guardrails.py +++ b/flowvault/samples/detect_api/check_guardrails.py @@ -36,6 +36,7 @@ def perform_check_guardrails(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/flowvault/samples/detect_api/deidentify_file.py b/flowvault/samples/detect_api/deidentify_file.py index 77375643..5edcb528 100644 --- a/flowvault/samples/detect_api/deidentify_file.py +++ b/flowvault/samples/detect_api/deidentify_file.py @@ -34,6 +34,7 @@ def perform_deidentify_file_submit_only(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) @@ -75,6 +76,7 @@ def perform_deidentify_file_poll(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) @@ -110,6 +112,7 @@ def perform_deidentify_file_presigned(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/flowvault/samples/detect_api/deidentify_string.py b/flowvault/samples/detect_api/deidentify_string.py index c057b358..84165c6d 100644 --- a/flowvault/samples/detect_api/deidentify_string.py +++ b/flowvault/samples/detect_api/deidentify_string.py @@ -36,6 +36,7 @@ def perform_deidentify_string(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) @@ -85,6 +86,7 @@ def perform_deidentify_string_inline(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/flowvault/samples/detect_api/get_run.py b/flowvault/samples/detect_api/get_run.py index 10a39c2f..f68ae924 100644 --- a/flowvault/samples/detect_api/get_run.py +++ b/flowvault/samples/detect_api/get_run.py @@ -35,6 +35,7 @@ def perform_get_run(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/flowvault/samples/detect_api/reidentify_file.py b/flowvault/samples/detect_api/reidentify_file.py index 01f4b994..08dc16c1 100644 --- a/flowvault/samples/detect_api/reidentify_file.py +++ b/flowvault/samples/detect_api/reidentify_file.py @@ -49,6 +49,7 @@ def perform_reidentify_file(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/flowvault/samples/detect_api/reidentify_string.py b/flowvault/samples/detect_api/reidentify_string.py index 045e35e1..716e5ad4 100644 --- a/flowvault/samples/detect_api/reidentify_string.py +++ b/flowvault/samples/detect_api/reidentify_string.py @@ -59,6 +59,7 @@ def perform_reidentify_string(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/skyvault/README.md b/skyvault/README.md index 9ce5a7e3..e58fc21e 100644 --- a/skyvault/README.md +++ b/skyvault/README.md @@ -2,7 +2,7 @@ > **This is the current, recommended version of the Skyflow SDK.** V2.1.0 brings flexible auth, multi-vault support, native data types, and rich error diagnostics. > -> Migrating from v1? See the **[Migration Guide](https://github.com/skyflowapi/skyflow-python/blob/main/docs/migrate_to_v2.md)** for step-by-step instructions. V1 is in maintenance mode and will reach End of Life on October 31, 2026. +> Migrating from v1? See the **[Migration Guide](https://github.com/skyflowapi/skyflow-python/blob/main/docs/migrate_to_v2.md)** for step-by-step instructions. V1 reached End of Life on October 31, 2026 and is no longer supported. The Skyflow Python SDK is designed to help with integrating Skyflow into a Python backend. @@ -552,7 +552,7 @@ Tokenization result: TokenizeResponse(tokenized_fields=[{'token': 'a1b2c3d4-...' ## Detect -Skyflow Detect finds and de-identifies sensitive data in strings and files, re-identifies tokens on demand, and screens text against guardrails. The SDK exposes the Detect V2 API through `DetectController`, returned by `skyflow_client.detect()` and `skyflow_client.detect('')`. Every request is scoped to that vault, and the controller shares the same credentials, timeouts and retries as `vault()`; the `vault_id` is filled in from the vault config for every method. +Skyflow Detect finds and de-identifies sensitive data in strings and files, re-identifies tokens on demand, and screens text against guardrails. The SDK exposes the Detect API through `DetectController`, returned by `skyflow_client.detect()` and `skyflow_client.detect('')`. Every request is scoped to that vault, and the controller shares the same credentials, timeouts and retries as `vault()`; the `vault_id` is filled in from the vault config for every method. Detect configurations are created in Studio or through the management API — the SDK does not create, update or delete them. Import request/response models from `skyflow.vault.detect` and enums from `skyflow.utils.enums`. @@ -677,6 +677,8 @@ configuration = DetectConfiguration( De-identifies a file. The API is asynchronous: the submit call returns a run id and the result is retrieved from the run. Build the request with `DeidentifyFileRequest`. +> **⚠️ Behaviour change from Detect V1:** `deidentify_file` **no longer waits for the result by default.** Without `poll_options` it submits and returns immediately — `status` is `None`, `output` is empty, and nothing is written to disk. Pass `poll_options` to block until the run finishes, or retrieve the result later with [`get_run`](#get-run-get_runrequest). Code ported from the old blocking behaviour runs without error but produces no outputs, so the failure surfaces downstream rather than at the call site. + Supply the file in exactly one of three ways: `file_path`, `file`, or `value` plus a `DeidentifySourceType` `data_source` (`BASE64`, the `SKYFLOW_ID` of a file record in the vault, or `PRESIGNED_URL`). A local `file` / `file_path` is read when the request is sent, base64-encoded, and its `data_format` inferred from the extension. Passing more than one of them, or none, fails client-side validation before any network call. `configuration_id` / `configuration` follow the same rules as for strings (see [Before you start](#before-you-start)). - **Submit only** (default): one call, returns `run_id`. `status` is `None` and nothing is written to disk — retrieve the result later with [`get_run`](#get-run-get_runrequest). @@ -769,13 +771,31 @@ Synchronous. Re-identification looks tokens up in the vault, so it applies only Build the request with `ReidentifyStringRequest`: the tokenized text and an optional list of `RedactionLevel` rules. On skyvault vaults each rule names an `entity_name` and a `redaction_type` — `PLAIN_TEXT` reveals the original value, `MASKED` masks it, `REDACTED` keeps the token, `DEFAULT` applies the vault's default. Without rules every token is replaced by its original value. +Only text that contains vault tokens can be re-identified, so the example is a round trip: de-identify with `VAULT_TOKEN` (which stores each value in a `table.column`), then re-identify the `processed_text` that comes back. A vault token is an opaque value like `a1b2c3d4-...`, not the `[NAME_1]` counter placeholder produced by `ENTITY_UNIQUE_COUNTER`. + ```python -from skyflow.vault.detect import ReidentifyStringRequest, RedactionLevel -from skyflow.utils.enums import EntityType, RedactionType +from skyflow.vault.detect import ( + DeidentifyStringRequest, DetectConfiguration, Detect, Entity, + ReidentifyStringRequest, RedactionLevel, +) +from skyflow.utils.enums import EntityType, DeidentificationType, RedactionType + +# 1. De-identify with VAULT_TOKEN so each value is stored in a table.column and can be re-identified. +deidentified = skyflow_client.detect().deidentify_string( + DeidentifyStringRequest( + text='My name is John Doe and my email is johndoe@acme.com.', + configuration=DetectConfiguration(detect=Detect(entities=[ + Entity(entity_type=EntityType.NAME, deidentification_type=DeidentificationType.VAULT_TOKEN, destination='persons.name'), + Entity(entity_type=EntityType.EMAIL_ADDRESS, deidentification_type=DeidentificationType.VAULT_TOKEN, destination='persons.email'), + ])), + ) +) +# deidentified.processed_text now holds vault tokens, e.g. 'My name is a1b2c3d4-... and my email is e5f6a7b8-...' +# 2. Re-identify those tokens; on skyvault each rule names an entity_name + redaction_type. response = skyflow_client.detect().reidentify_string( ReidentifyStringRequest( - text='My name is [NAME_1] and my email is [EMAIL_ADDRESS_1].', + text=deidentified.processed_text, redaction_level=[ RedactionLevel(entity_name=EntityType.NAME, redaction_type=RedactionType.PLAIN_TEXT), RedactionLevel(entity_name=EntityType.EMAIL_ADDRESS, redaction_type=RedactionType.MASKED), @@ -799,6 +819,7 @@ import base64 from skyflow.vault.detect import ReidentifyFileRequest, RedactionLevel from skyflow.utils.enums import ReidentifySourceType, FileDataFormat, EntityType, RedactionType, DetectRunStatus +data_format = FileDataFormat.TXT with open('/data/redacted/notes.txt', 'rb') as handle: encoded = base64.b64encode(handle.read()).decode('utf-8') @@ -806,23 +827,26 @@ response = skyflow_client.detect().reidentify_file( ReidentifyFileRequest( data_source=ReidentifySourceType.BASE64, value=encoded, - data_format=FileDataFormat.TXT, + data_format=data_format, redaction_level=[RedactionLevel(entity_name=EntityType.NAME, redaction_type=RedactionType.PLAIN_TEXT)], ) ) if response.status == DetectRunStatus.SUCCESS: for index, output in enumerate(response.output): - # output.processed_file is base64 or a presigned URL, per response.output_type - with open(f'reidentified-{index}.txt', 'wb') as out: - out.write(base64.b64decode(output.processed_file)) + # Branch on output_type: base64 bytes are decoded, a presigned URL is downloaded. + if response.output_type == ReidentifySourceType.BASE64: + with open(f'reidentified-{index}.{data_format.value}', 'wb') as out: + out.write(base64.b64decode(output.processed_file)) + else: # ReidentifySourceType.PRESIGNED_URL — processed_file is a URL to fetch + print('download re-identified file from', output.processed_file) ``` -Returns a `ReidentifyFileResponse` with `status`, `output_type`, `output` (a list of `FileOutput`) and `metrics`. +Returns a `ReidentifyFileResponse` with `status`, `output_type` (a `ReidentifySourceType`), `output` (a list of `FileOutput`) and `metrics`. ### Check Guardrails: `.check_guardrails(request)` -Synchronous. Screens text for toxicity and a list of denied topics — a natural pre-LLM check. Build the request with `CheckGuardrailsRequest`. `check_toxicity` and `deny_topics` are optional; the SDK sends each only when set and fills in no default. The response is passed through as the API returns it. +Synchronous. Screens text for toxicity and a list of denied topics — a natural pre-LLM check. Build the request with `CheckGuardrailsRequest`. `check_toxicity` and `deny_topics` are optional; the SDK sends each only when set and fills in no default. **When `check_toxicity` is omitted, the server still checks toxicity by default.** The response is passed through as the API returns it. ```python from skyflow.vault.detect import CheckGuardrailsRequest diff --git a/skyvault/samples/detect_api/check_guardrails.py b/skyvault/samples/detect_api/check_guardrails.py index 070d69fa..ce4b5db5 100644 --- a/skyvault/samples/detect_api/check_guardrails.py +++ b/skyvault/samples/detect_api/check_guardrails.py @@ -36,6 +36,7 @@ def perform_check_guardrails(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/skyvault/samples/detect_api/deidentify_file.py b/skyvault/samples/detect_api/deidentify_file.py index 6940345f..3aad5a1d 100644 --- a/skyvault/samples/detect_api/deidentify_file.py +++ b/skyvault/samples/detect_api/deidentify_file.py @@ -51,6 +51,7 @@ def perform_deidentify_file(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/skyvault/samples/detect_api/deidentify_string.py b/skyvault/samples/detect_api/deidentify_string.py index fbbbdd0f..def6c3dd 100644 --- a/skyvault/samples/detect_api/deidentify_string.py +++ b/skyvault/samples/detect_api/deidentify_string.py @@ -36,6 +36,7 @@ def perform_deidentify_string(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) @@ -78,6 +79,7 @@ def perform_deidentify_string_inline(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/skyvault/samples/detect_api/get_run.py b/skyvault/samples/detect_api/get_run.py index 9ce02d67..b4f69810 100644 --- a/skyvault/samples/detect_api/get_run.py +++ b/skyvault/samples/detect_api/get_run.py @@ -37,6 +37,7 @@ def perform_get_run(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/skyvault/samples/detect_api/reidentify_file.py b/skyvault/samples/detect_api/reidentify_file.py index 9bc71148..5ba7942f 100644 --- a/skyvault/samples/detect_api/reidentify_file.py +++ b/skyvault/samples/detect_api/reidentify_file.py @@ -46,6 +46,7 @@ def perform_reidentify_file(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/skyvault/samples/detect_api/reidentify_string.py b/skyvault/samples/detect_api/reidentify_string.py index b685da1f..23479ec4 100644 --- a/skyvault/samples/detect_api/reidentify_string.py +++ b/skyvault/samples/detect_api/reidentify_string.py @@ -37,6 +37,7 @@ def perform_reidentify_string(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, })