Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changeset/safe-upstream-retries.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
"@executor-js/sdk": patch
"@executor-js/plugin-openapi": patch
"@executor-js/plugin-graphql": patch
---

Mark safe upstream failures as retryable.
1 change: 1 addition & 0 deletions packages/core/sdk/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -422,6 +422,7 @@ export {
ToolResult,
annotateToolResultOutcome,
isToolFile,
isRetryableUpstreamFailure,
isToolResult,
type ToolFile,
type ToolFile as ToolFileValue,
Expand Down
69 changes: 68 additions & 1 deletion packages/core/sdk/src/tool-result.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { describe, expect, it } from "@effect/vitest";
import { ToolResult, isToolFile, isToolResult } from "./tool-result";
import { ToolResult, isRetryableUpstreamFailure, isToolFile, isToolResult } from "./tool-result";

describe("ToolResult", () => {
it("ok wraps a value", () => {
Expand Down Expand Up @@ -49,6 +49,73 @@ describe("ToolResult", () => {
});
});

describe("isRetryableUpstreamFailure", () => {
it.each([
{
name: "408 is retryable for a non-idempotent operation",
input: { status: 408, idempotent: false },
expected: true,
},
{
name: "425 is retryable for a non-idempotent operation",
input: { status: 425, idempotent: false },
expected: true,
},
{
name: "429 is retryable for a non-idempotent operation",
input: { status: 429, idempotent: false },
expected: true,
},
{
name: "503 is retryable for a non-idempotent operation",
input: { status: 503, idempotent: false },
expected: true,
},
{
name: "500 is retryable for an idempotent operation",
input: { status: 500, idempotent: true },
expected: true,
},
{
name: "502 is retryable for an idempotent operation",
input: { status: 502, idempotent: true },
expected: true,
},
{
name: "504 is retryable for an idempotent operation",
input: { status: 504, idempotent: true },
expected: true,
},
{
name: "500 is not retryable for a non-idempotent operation",
input: { status: 500, idempotent: false },
expected: false,
},
{
name: "404 is not retryable for an idempotent operation",
input: { status: 404, idempotent: true },
expected: false,
},
{
name: "401 is not retryable for an idempotent operation",
input: { status: 401, idempotent: true },
expected: false,
},
{
name: "a timeout is retryable for an idempotent operation",
input: { timedOut: true, idempotent: true },
expected: true,
},
{
name: "a timeout is not retryable for a non-idempotent operation",
input: { timedOut: true, idempotent: false },
expected: false,
},
])("$name", ({ input, expected }) => {
expect(isRetryableUpstreamFailure(input)).toBe(expected);
});
});

describe("ToolFile", () => {
it("accepts a base64 file value", () => {
const file = {
Expand Down
22 changes: 22 additions & 0 deletions packages/core/sdk/src/tool-result.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,28 @@ export const ToolErrorSchema = Schema.Struct({

export type ToolError = typeof ToolErrorSchema.Type;

export const isRetryableUpstreamFailure = (input: {
readonly status?: number;
readonly timedOut?: boolean;
readonly idempotent: boolean;
}): boolean => {
if (
input.status === 408 ||
input.status === 425 ||
input.status === 429 ||
input.status === 503
) {
return true;
}
return (
input.idempotent &&
(input.timedOut === true ||
input.status === 500 ||
input.status === 502 ||
input.status === 504)
);
};

export const ToolHttpMetaSchema = Schema.Struct({
status: Schema.Number,
headers: Schema.Record(Schema.String, Schema.String),
Expand Down
27 changes: 21 additions & 6 deletions packages/plugins/graphql/src/sdk/plugin.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -437,7 +437,7 @@ describe("graphqlPlugin real protocol server", () => {
}),
);

it.effect("surfaces non-2xx invocation responses as ToolResult.fail", () =>
it.effect("marks only idempotent non-2xx invocation responses retryable", () =>
Effect.gen(function* () {
const server = yield* serveTestHttpApp((request) =>
Effect.gen(function* () {
Expand All @@ -447,7 +447,7 @@ describe("graphqlPlugin real protocol server", () => {
return HttpServerResponse.jsonUnsafe({ data: introspectionResult });
}
return HttpServerResponse.text("temporary upstream outage", {
status: 503,
status: body.includes("mutation") ? 500 : 503,
contentType: "text/plain",
});
}),
Expand All @@ -465,18 +465,33 @@ describe("graphqlPlugin real protocol server", () => {
value: "unused",
});

const result = yield* executor.execute(toolAddr("http_error_graph", "main", "query.hello"), {
name: "Ada",
});
const queryResult = yield* executor.execute(
toolAddr("http_error_graph", "main", "query.hello"),
{ name: "Ada" },
);
const mutationResult = yield* executor.execute(
toolAddr("http_error_graph", "main", "mutation.setGreeting"),
{ message: "hi" },
);

expect(result).toMatchObject({
expect(queryResult).toMatchObject({
ok: false,
error: {
code: "graphql_http_error",
status: 503,
message: "GraphQL request failed with HTTP 503",
retryable: true,
},
});
expect(mutationResult).toMatchObject({
ok: false,
error: {
code: "graphql_http_error",
status: 500,
message: "GraphQL request failed with HTTP 500",
},
});
expect(mutationResult).not.toHaveProperty("error.retryable");
}),
);

Expand Down
7 changes: 7 additions & 0 deletions packages/plugins/graphql/src/sdk/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
IntegrationAlreadyExistsError,
IntegrationDetectionResult,
IntegrationSlug,
isRetryableUpstreamFailure,
mergeAuthTemplates,
sha256Hex,
ToolName,
Expand Down Expand Up @@ -1445,6 +1446,12 @@ export const graphqlPlugin = definePlugin((options?: GraphqlPluginOptions) => {
data: result.data,
errors: result.errors,
},
...(isRetryableUpstreamFailure({
status: result.status,
idempotent: op.binding.kind === "query",
})
? { retryable: true }
: {}),
});
}
return ToolResult.ok(result.data);
Expand Down
11 changes: 11 additions & 0 deletions packages/plugins/openapi/src/sdk/backing.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
authToolFailure,
classifyHttpStatus,
detectInsufficientScope,
isRetryableUpstreamFailure,
sortHealthCheckCandidatesByIdentity,
extractIdentity,
extractResponseFields,
Expand Down Expand Up @@ -696,6 +697,7 @@ export const invokeOpenApiBackedTool = (input: {
Object.assign(queryParams, rendered.queryParams);
}

const idempotent = ["get", "head", "options", "put", "delete"].includes(binding.method);
const invocation = yield* invokeWithLayer(
binding,
(input.args ?? {}) as Record<string, unknown>,
Expand All @@ -714,6 +716,9 @@ export const invokeOpenApiBackedTool = (input: {
code: "upstream_response_headers_timeout",
message: error.message,
details: error.cause ?? error,
...(isRetryableUpstreamFailure({ timedOut: true, idempotent })
? { retryable: true }
: {}),
}),
})
: error.reason === "response_body_timeout"
Expand All @@ -723,6 +728,9 @@ export const invokeOpenApiBackedTool = (input: {
code: "upstream_response_body_timeout",
message: error.message,
details: error.cause ?? error,
...(isRetryableUpstreamFailure({ timedOut: true, idempotent })
? { retryable: true }
: {}),
}),
})
: Effect.fail(error),
Expand Down Expand Up @@ -787,6 +795,9 @@ export const invokeOpenApiBackedTool = (input: {
status: result.status,
message: extractOpenApiUpstreamMessage(result.error, result.status),
details: result.error,
...(isRetryableUpstreamFailure({ status: result.status, idempotent })
? { retryable: true }
: {}),
});
}
return ToolResult.ok(result.data, {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ describe("OpenAPI response body timeout", () => {
ok: false,
error: {
code: "upstream_response_body_timeout",
retryable: true,
message: expect.stringContaining("response body"),
},
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ describe("OpenAPI response headers timeout", () => {
ok: false,
error: {
code: "upstream_response_headers_timeout",
retryable: true,
message: expect.stringContaining("Upstream returned no response headers within 100ms"),
},
});
Expand Down
59 changes: 52 additions & 7 deletions packages/plugins/openapi/src/sdk/upstream-failures.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import {
AuthTemplateSlug,
ConnectionName,
IntegrationSlug,
isToolResult,
ToolAddress,
} from "@executor-js/sdk";
import { makeTestConfig, memoryCredentialsPlugin } from "@executor-js/sdk/testing";
Expand All @@ -49,6 +50,7 @@ const testPlugins = () =>
// `/things` GET op `listThings` under group "things" → tool path
// `things.listThings`, used verbatim (dots and all) as the address tool segment.
const LIST_THINGS = "things.listThings";
const CREATE_THING = "things.createThing";

type ResponseScript = (req: {
url: string;
Expand All @@ -63,8 +65,8 @@ type ResponseScript = (req: {
const startScriptedServer = (script: ResponseScript) =>
serveOpenApiHttpApiTestServer({
api: FailureApi,
handlersLayer: HttpApiBuilder.group(FailureApi, "things", (handlers) =>
handlers.handle("listThings", () =>
handlersLayer: HttpApiBuilder.group(FailureApi, "things", (handlers) => {
const handleRequest = () =>
Effect.gen(function* () {
const request = yield* HttpServerRequest.HttpServerRequest;
const result = script({
Expand All @@ -76,9 +78,9 @@ const startScriptedServer = (script: ResponseScript) =>
status: result.status ?? 200,
headers: result.headers ?? { "content-type": "application/json" },
});
}),
),
),
});
return handlers.handle("listThings", handleRequest).handle("createThing", handleRequest);
}),
});

const startDroppingServer = () =>
Expand Down Expand Up @@ -106,6 +108,9 @@ const ThingsGroup = HttpApiGroup.make("things").add(
HttpApiEndpoint.get("listThings", "/things", {
success: Schema.Array(Schema.Record(Schema.String, Schema.Unknown)),
}),
HttpApiEndpoint.post("createThing", "/things", {
success: Schema.Array(Schema.Record(Schema.String, Schema.Unknown)),
}),
);

const FailureApi = HttpApi.make("failuresTest")
Expand All @@ -131,11 +136,11 @@ const buildExecutor = (baseUrl: string) =>
return { executor, address };
});

const buildExecutorForOpenApiServer = (server: OpenApiTestServerShape) =>
const buildExecutorForOpenApiServer = (server: OpenApiTestServerShape, toolName = LIST_THINGS) =>
Effect.gen(function* () {
const executor = yield* createExecutor(makeTestConfig({ plugins: testPlugins() }));
const conn = yield* addOpenApiTestConnection(executor, server, { slug: "f" });
return { executor, address: conn.address(LIST_THINGS) };
return { executor, address: conn.address(toolName) };
});

describe("OpenAPI upstream failure modes", () => {
Expand Down Expand Up @@ -171,6 +176,45 @@ describe("OpenAPI upstream failure modes", () => {
}),
);

it.effect("marks only safe upstream HTTP failures retryable", () =>
Effect.gen(function* () {
const cases = [
{ method: "POST", status: 429, toolName: CREATE_THING, retryable: true },
{ method: "POST", status: 500, toolName: CREATE_THING, retryable: false },
{ method: "GET", status: 500, toolName: LIST_THINGS, retryable: true },
{ method: "GET", status: 404, toolName: LIST_THINGS, retryable: false },
] as const;

for (const testCase of cases) {
const observedMethods: string[] = [];
const server = yield* startScriptedServer(({ method }) => {
observedMethods.push(method);
return { status: testCase.status };
});
const { executor, address } = yield* buildExecutorForOpenApiServer(
server,
testCase.toolName,
);

const result = yield* executor.execute(address, {});

expect(observedMethods).toEqual([testCase.method]);
expect(result).toMatchObject({
ok: false,
error: {
code: "upstream_http_error",
status: testCase.status,
...(testCase.retryable ? { retryable: true } : {}),
},
});
const error = isToolResult(result) && !result.ok ? result.error : undefined;
expect(error).toBeDefined();
expect(Object.hasOwn(error ?? {}, "retryable")).toBe(testCase.retryable);
expect(error?.retryable).toBe(testCase.retryable ? true : undefined);
}
}),
);

it.effect("upstream 4xx surfaces structured error body", () =>
Effect.gen(function* () {
const server = yield* startScriptedServer(() => ({
Expand Down Expand Up @@ -207,6 +251,7 @@ describe("OpenAPI upstream failure modes", () => {
error: {
code: "connection_rejected",
status: 401,
retryable: false,
message: expect.stringContaining("Upstream rejected credentials"),
details: {
category: "authentication",
Expand Down