From b64b78a09814f41e0dad46d51d07c4217cb94832 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 13 Oct 2026 13:10:16 -0700 Subject: [PATCH 1/3] improvement(search): bound the any-term fallback so a slow relaxed query keeps the strict page When an all-terms Atlassian search underfills, the live search runs a relaxed any-term query. For Jira that query ORs every pair of up to 8 terms (28 pairs, 56 `text ~` clauses), which can run past the 10 s HTTP timeout on a large site. The transport failure was rethrown, so the whole account leg reported unavailable: the strict matches already fetched were discarded and the search waited out the timeout. - The relaxed search gets its own 4 s budget, composed with the caller's signal through AbortSignal.any and passed down as a per-search signal (NativeSearchInput.signal -> searchAtlassian -> client.json -> secureFetchWithValidation), so the HTTP request itself is cancelled rather than raced. The account leg already has a 12 s deadline that covers resolve, session, the strict search, verification, and the 2.5 s preview reads; a relaxed query normally takes ~1.5 s, so 4 s leaves headroom for a slow but healthy query while ending a hung one with time left for verification and previews inside the leg deadline. Composing with the leg signal means it can never run past that deadline. - Any relaxed failure other than the caller's own cancellation (provider refusal, the budget running out, a transport error) keeps the strict page, marked partial with a message, the same shape as the existing refusal path. Caller cancellation still propagates. A successful relaxed search behaves exactly as before. - Jira relaxed JQL pairs at most the six longest distinct terms (at most 15 pairs), and a word and its regular plural share one slot, since Jira stems text search. Ranking by term coverage still uses every significant term, and Confluence's siteSearch clause is unchanged. - A memoized GET is never shared with a request that carries its own signal. --- .../lib/sim-search/live/account-session.ts | 2 +- apps/sim/lib/sim-search/live/atlassian.ts | 2 + apps/sim/lib/sim-search/live/http.ts | 8 +- .../lib/sim-search/live/term-fallback.test.ts | 127 +++++++++++++++--- apps/sim/lib/sim-search/live/term-fallback.ts | 62 +++++++-- .../lib/sim-search/live/text-match.test.ts | 22 +++ apps/sim/lib/sim-search/live/text-match.ts | 40 ++++-- apps/sim/lib/sim-search/live/types.ts | 7 + 8 files changed, 226 insertions(+), 44 deletions(-) diff --git a/apps/sim/lib/sim-search/live/account-session.ts b/apps/sim/lib/sim-search/live/account-session.ts index 0092b4cff55..c343e894fd1 100644 --- a/apps/sim/lib/sim-search/live/account-session.ts +++ b/apps/sim/lib/sim-search/live/account-session.ts @@ -206,7 +206,7 @@ export async function openLiveAccountSession( ) } return client && requiresEveryTerm(provider) - ? searchWithAnyTermFallback(searchOnce, search) + ? searchWithAnyTermFallback(searchOnce, search, signal) : searchOnce(search) }, async verify(document) { diff --git a/apps/sim/lib/sim-search/live/atlassian.ts b/apps/sim/lib/sim-search/live/atlassian.ts index b7e0cfc93e8..b80f4d87079 100644 --- a/apps/sim/lib/sim-search/live/atlassian.ts +++ b/apps/sim/lib/sim-search/live/atlassian.ts @@ -169,6 +169,7 @@ export async function searchAtlassian( fields: ['summary', 'description', 'updated', 'creator', 'status'], ...(input.native?.cursor && single ? { nextPageToken: input.native.cursor } : {}), }, + signal: input.signal, }) ) return { @@ -190,6 +191,7 @@ export async function searchAtlassian( expand: 'content.version,content.space', ...(input.native?.cursor && single ? { cursor: input.native.cursor } : {}), }, + signal: input.signal, }) ) const next = string(object(data._links).next) diff --git a/apps/sim/lib/sim-search/live/http.ts b/apps/sim/lib/sim-search/live/http.ts index 81f20d50654..8d9ebf4fa1a 100644 --- a/apps/sim/lib/sim-search/live/http.ts +++ b/apps/sim/lib/sim-search/live/http.ts @@ -39,9 +39,11 @@ export function createNativeClient(input: { query?: Record body?: unknown googleService?: 'sheets' + signal?: AbortSignal } ) { - input.signal.throwIfAborted() + const signal = options?.signal ? AbortSignal.any([input.signal, options.signal]) : input.signal + signal.throwIfAborted() if (++requests > (input.requestBudget ?? NATIVE_SEARCH_REQUEST_BUDGET)) throw new NativeSearchError('unavailable', 'Request budget reached. Narrow the query.') const url = new URL(input.origin) @@ -68,7 +70,7 @@ export function createNativeClient(input: { }, method: options?.body ? 'POST' : 'GET', ...(options?.body ? { body: JSON.stringify(options.body) } : {}), - signal: input.signal, + signal, timeout: 10_000, maxResponseBytes: NATIVE_RESPONSE_MAX_BYTES, maxRedirects: 0, @@ -144,7 +146,7 @@ export function withJsonMemo(client: NativeClient): NativeClient { ...client, json(...request) { const [path, options] = request - if (!options?.memo || options.body) return client.json(...request) + if (!options?.memo || options.body || options.signal) return client.json(...request) const key = JSON.stringify([path, options.query, options.googleService]) let pending = memo.get(key) if (!pending) { diff --git a/apps/sim/lib/sim-search/live/term-fallback.test.ts b/apps/sim/lib/sim-search/live/term-fallback.test.ts index 1284af22a1e..34c1536789f 100644 --- a/apps/sim/lib/sim-search/live/term-fallback.test.ts +++ b/apps/sim/lib/sim-search/live/term-fallback.test.ts @@ -1,10 +1,30 @@ -import { describe, expect, it, vi } from 'vitest' -import { NativeSearchError } from '@/lib/sim-search/live/http' +import { + inputValidationMock, + inputValidationMockFns, +} from '@sim/testing/mocks/input-validation.mock' +import { afterEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + +import { searchAtlassian } from '@/lib/sim-search/live/atlassian' +import { createNativeClient, NativeSearchError } from '@/lib/sim-search/live/http' import { searchWithAnyTermFallback } from '@/lib/sim-search/live/term-fallback' import type { NativeDocument, NativePage, NativeSearchInput } from '@/lib/sim-search/live/types' const input: NativeSearchInput = { query: 'contractor offboarding runbook', limit: 4, scopes: [] } +/** The any-term search's own budget, which a slow provider must not outlast. */ +const BUDGET_MS = 4000 + +/** A caller that never cancels. */ +const live = new AbortController().signal + +/** Settles only when its request is cancelled, like a provider that never answers. */ +const untilAborted = (signal: AbortSignal | undefined) => + new Promise((_, reject) => { + signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + function page(...documents: Partial[]): NativePage { return { documents: documents.map((document, index) => ({ @@ -26,6 +46,10 @@ function provider(strict: NativePage, relaxed: NativePage | Error) { } describe('searchWithAnyTermFallback', () => { + afterEach(() => { + vi.useRealTimers() + }) + it('follows an exhausted all-terms page with partial matches, most terms first', async () => { const search = provider( page({ id: 'exact', title: 'Contractor offboarding runbook' }), @@ -37,7 +61,7 @@ describe('searchWithAnyTermFallback', () => { { id: 'contractors', title: 'Contractor directory' } ) ) - const result = await searchWithAnyTermFallback(search, input) + const result = await searchWithAnyTermFallback(search, input, live) expect(result.documents.map((document) => document.id)).toEqual([ 'exact', 'in-title', @@ -45,6 +69,7 @@ describe('searchWithAnyTermFallback', () => { 'runbook', ]) expect(result.nextCursor).toBeUndefined() + expect(result.partial).toBeFalsy() expect(result.message).toContain('Few results contained every search term') }) @@ -58,7 +83,7 @@ describe('searchWithAnyTermFallback', () => { ? { documents: page(...loose).documents.slice(0, request.limit) } : page() ) - const result = await searchWithAnyTermFallback(search, { ...input, limit: 2 }) + const result = await searchWithAnyTermFallback(search, { ...input, limit: 2 }, live) expect(result.documents.map((document) => document.title)).toEqual([ 'Contractor offboarding form', 'Runbook 0', @@ -68,34 +93,36 @@ describe('searchWithAnyTermFallback', () => { it('adds a few partial matches however large the page', async () => { const partial = Array.from({ length: 12 }, () => ({ title: 'Contractor offboarding' })) const search = provider(page(), page(...partial)) - const result = await searchWithAnyTermFallback(search, { ...input, limit: 20 }) + const result = await searchWithAnyTermFallback(search, { ...input, limit: 20 }, live) expect(result.documents).toHaveLength(5) }) it('never relaxes a native query, which is the caller’s exact syntax', async () => { const search = provider(page(), page({ id: 'loose' })) const native = { provider: 'confluence' as const, query: 'text ~ "runbook"' } - expect((await searchWithAnyTermFallback(search, { ...input, native })).documents).toEqual([]) + expect((await searchWithAnyTermFallback(search, { ...input, native }, live)).documents).toEqual( + [] + ) expect(search).toHaveBeenCalledTimes(1) }) it('keeps a continuable page so its cursor continues the query that produced it', async () => { const strict = { ...page({ id: 'exact' }), nextCursor: 'page-2' } const search = provider(strict, page({ id: 'loose' })) - expect(await searchWithAnyTermFallback(search, input)).toBe(strict) + expect(await searchWithAnyTermFallback(search, input, live)).toBe(strict) expect(search).toHaveBeenCalledTimes(1) }) it('does not relax a page that is already full', async () => { const strict = page({}, {}, {}, {}) const search = provider(strict, page({ id: 'loose' })) - expect(await searchWithAnyTermFallback(search, input)).toBe(strict) + expect(await searchWithAnyTermFallback(search, input, live)).toBe(strict) expect(search).toHaveBeenCalledTimes(1) }) it('does not relax a query with fewer than two distinctive terms', async () => { const search = provider(page(), page({ id: 'loose' })) - await searchWithAnyTermFallback(search, { ...input, query: 'what is the runbook' }) + await searchWithAnyTermFallback(search, { ...input, query: 'what is the runbook' }, live) expect(search).toHaveBeenCalledTimes(1) }) @@ -104,25 +131,95 @@ describe('searchWithAnyTermFallback', () => { page({ id: 'exact' }), new NativeSearchError('rate_limited', 'Slow down') ) - const result = await searchWithAnyTermFallback(search, input) + const result = await searchWithAnyTermFallback(search, input, live) expect(result.documents.map((document) => document.id)).toEqual(['exact']) expect(result).toMatchObject({ partial: true, message: 'Slow down' }) }) - it('propagates cancellation during the any-term search instead of returning a page', async () => { - const search = provider(page({ id: 'exact' }), new DOMException('Timed out', 'TimeoutError')) - await expect(searchWithAnyTermFallback(search, input)).rejects.toThrow('Timed out') + it('propagates the caller’s cancellation during the any-term search instead of returning a page', async () => { + const caller = new AbortController() + const search = vi.fn(async (request: NativeSearchInput) => + request.textMatch === 'any' ? untilAborted(request.signal) : page({ id: 'exact' }) + ) + const result = searchWithAnyTermFallback(search, input, caller.signal) + await vi.waitFor(() => expect(search).toHaveBeenCalledTimes(2)) + caller.abort(new DOMException('Cancelled', 'AbortError')) + await expect(result).rejects.toThrow('Cancelled') + }) + + it('keeps the all-terms page as partial when the any-term search outlasts its budget', async () => { + vi.useFakeTimers() + let relaxedSignal: AbortSignal | undefined + const search = vi.fn(async (request: NativeSearchInput) => { + if (request.textMatch !== 'any') return page({ id: 'exact' }) + relaxedSignal = request.signal + return untilAborted(request.signal) + }) + let settled = false + const result = searchWithAnyTermFallback(search, input, live).finally(() => { + settled = true + }) + await vi.advanceTimersByTimeAsync(BUDGET_MS - 1) + expect(settled).toBe(false) + await vi.advanceTimersByTimeAsync(1) + const searched = await result + expect(relaxedSignal?.aborted).toBe(true) + expect(searched.documents.map((document) => document.id)).toEqual(['exact']) + expect(searched).toMatchObject({ partial: true }) + expect(searched.message).toContain('some of the search terms') + }) + + it('keeps the all-terms page as partial when the any-term search fails in transport', async () => { + const search = provider(page({ id: 'exact' }), new TypeError('fetch failed')) + const searched = await searchWithAnyTermFallback(search, input, live) + expect(searched.documents.map((document) => document.id)).toEqual(['exact']) + expect(searched).toMatchObject({ partial: true }) + expect(searched.message).toContain('some of the search terms') + }) + + it('cancels a slow relaxed Jira request at the network once the budget runs out', async () => { + vi.useFakeTimers() + const relaxedSignals: AbortSignal[] = [] + inputValidationMockFns.mockSecureFetchWithValidation.mockImplementation( + async (url: string, init: { body?: string; signal: AbortSignal }) => { + if (url.endsWith('/oauth/token/accessible-resources')) + return Response.json([{ id: 'cloud', url: 'https://acme.atlassian.net' }]) + const { jql } = JSON.parse(init.body ?? '{}') as { jql: string } + if (!jql.includes(' OR ')) + return Response.json({ issues: [{ key: 'ENG-1', fields: { summary: 'Exact' } }] }) + relaxedSignals.push(init.signal) + return untilAborted(init.signal) + } + ) + const caller = new AbortController() + const client = createNativeClient({ + origin: 'https://api.atlassian.com', + accessToken: 'private', + signal: caller.signal, + }) + const result = searchWithAnyTermFallback( + (request) => searchAtlassian(client, 'jira', request), + input, + caller.signal + ) + await vi.advanceTimersByTimeAsync(BUDGET_MS) + const searched = await result + expect(relaxedSignals).toHaveLength(1) + expect(relaxedSignals[0]?.aborted).toBe(true) + expect(caller.signal.aborted).toBe(false) + expect(searched.documents.map((document) => document.id)).toEqual(['ENG-1']) + expect(searched.partial).toBe(true) }) it('does not relax a date-sorted search, whose page is reordered by date', async () => { const search = provider(page(), page({ id: 'loose' })) - await searchWithAnyTermFallback(search, { ...input, filters: { sortBy: 'newest' } }) + await searchWithAnyTermFallback(search, { ...input, filters: { sortBy: 'newest' } }, live) expect(search).toHaveBeenCalledTimes(1) }) it('leaves the page untouched when the any-term search adds nothing new', async () => { const strict = page({ id: 'exact' }) const search = provider(strict, page({ id: 'exact' })) - expect(await searchWithAnyTermFallback(search, input)).toBe(strict) + expect(await searchWithAnyTermFallback(search, input, live)).toBe(strict) }) }) diff --git a/apps/sim/lib/sim-search/live/term-fallback.ts b/apps/sim/lib/sim-search/live/term-fallback.ts index 0ceb1b0990b..7d323bc4f6e 100644 --- a/apps/sim/lib/sim-search/live/term-fallback.ts +++ b/apps/sim/lib/sim-search/live/term-fallback.ts @@ -1,3 +1,5 @@ +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' import { dateSortDirection } from '@/lib/sim-search/live/dates' import { NativeSearchError } from '@/lib/sim-search/live/http' import { joinMessages } from '@/lib/sim-search/live/pages' @@ -13,6 +15,15 @@ const ANY_TERM_CANDIDATES = 25 /** Partial matches added to a page, so a loosely matching provider cannot crowd out other sources. */ const ANY_TERM_RESULTS = 5 +/** + * Time the any-term search may take before the all-terms page is returned without it. It runs + * after the all-terms search inside the account's 12 s deadline and before the 2.5 s preview + * reads, and a provider's slow OR query must not cost the account the matches it already has. + */ +const ANY_TERM_BUDGET_MS = 4000 + +const logger = createLogger('SimSearchTermFallback') + /** One provider item, however many queries returned it. */ const documentKey = ({ container, kind, id }: NativeDocument) => JSON.stringify([container ?? '', kind ?? '', id]) @@ -26,13 +37,17 @@ const documentKey = ({ container, kind, id }: NativeDocument) => * ordered by how many terms their title and preview contain, then how many their title contains, * then provider order. The fallback runs only for a plain relevance-ordered query: native queries are * the caller's exact syntax, and a date-sorted page is reordered by date. It never replaces a - * continuable page, so a cursor always continues the query that produced it. A provider refusal - * keeps the all-terms page, reported as partial with the provider's message; cancellation and - * transport failures propagate like the first search. + * continuable page, so a cursor always continues the query that produced it. + * + * The any-term search gets its own budget, passed to the provider as the request's signal, so a + * slow OR query is cancelled rather than awaited. Whenever it fails short of the caller's own + * cancellation (a provider refusal, the budget running out, a transport error), the all-terms page + * is kept, reported as partial. Cancellation through `signal` propagates like the first search. */ export async function searchWithAnyTermFallback( search: (input: NativeSearchInput) => Promise, - input: NativeSearchInput + input: NativeSearchInput, + signal: AbortSignal ): Promise { const strict = await search(input) const terms = significantTerms(input.query) @@ -45,16 +60,35 @@ export async function searchWithAnyTermFallback( strict.documents.length >= input.limit ) return strict - const relaxed = await search({ - ...input, - limit: Math.max(input.limit, ANY_TERM_CANDIDATES), - textMatch: 'any', - }).catch((error: unknown) => { - if (error instanceof NativeSearchError) return error - throw error - }) - if (relaxed instanceof NativeSearchError) - return { ...strict, partial: true, message: joinMessages([strict.message, relaxed.message]) } + const budget = new AbortController() + const timer = setTimeout( + () => budget.abort(new DOMException('Any-term search budget exhausted', 'TimeoutError')), + ANY_TERM_BUDGET_MS + ) + let relaxed: NativePage + try { + relaxed = await search({ + ...input, + limit: Math.max(input.limit, ANY_TERM_CANDIDATES), + textMatch: 'any', + signal: AbortSignal.any([signal, budget.signal, ...(input.signal ? [input.signal] : [])]), + }) + } catch (error) { + signal.throwIfAborted() + input.signal?.throwIfAborted() + if (!(error instanceof NativeSearchError)) + logger.warn('Any-term search failed; keeping the all-terms page', { + timedOut: budget.signal.aborted, + error: getErrorMessage(error), + }) + const reason = + error instanceof NativeSearchError + ? error.message + : 'Results containing only some of the search terms could not be retrieved, so only results containing every term are shown.' + return { ...strict, partial: true, message: joinMessages([strict.message, reason]) } + } finally { + clearTimeout(timer) + } const seen = new Set(strict.documents.map(documentKey)) const additions = relaxed.documents .filter((document) => !seen.has(documentKey(document))) diff --git a/apps/sim/lib/sim-search/live/text-match.test.ts b/apps/sim/lib/sim-search/live/text-match.test.ts index 2b8e635a704..158a2fecb68 100644 --- a/apps/sim/lib/sim-search/live/text-match.test.ts +++ b/apps/sim/lib/sim-search/live/text-match.test.ts @@ -18,11 +18,33 @@ describe('text match clauses', () => { expect(jiraTextClause('API api rollout', 'any')).toBe('(text ~ "rollout" OR text ~ "API")') }) + it('pairs only the six longest distinct terms of a long query, so the JQL stays bounded', () => { + const clause = jiraTextClause( + 'rotate expired gateway credentials before quarterly compliance audit deadline', + 'any' + ) + expect(clause.match(/text ~/g)).toHaveLength(30) + for (const term of ['credentials', 'compliance', 'quarterly', 'deadline', 'expired', 'gateway']) + expect(clause).toContain(`"${term}"`) + expect(clause).not.toContain('"rotate"') + expect(clause).not.toContain('"audit"') + }) + + it('spends one pair slot on a word and its plural, which Jira stems alike', () => { + expect(jiraTextClause('token tokens rotation', 'any')).toBe( + '(text ~ "rotation" OR text ~ "tokens")' + ) + expect(jiraTextClause('processes process', 'any')).toBe('text ~ "processes process"') + }) + it('relaxes a Confluence query to its relevance-ranked site search', () => { expect(confluenceTextClause('release process', 'all')).toBe('text ~ "release process"') expect(confluenceTextClause('what is the release process', 'any')).toBe( 'siteSearch ~ "what is the release process"' ) + expect(confluenceTextClause('token tokens rotation', 'any')).toBe( + 'siteSearch ~ "token tokens rotation"' + ) }) it('escapes every term so a term cannot add query syntax', () => { diff --git a/apps/sim/lib/sim-search/live/text-match.ts b/apps/sim/lib/sim-search/live/text-match.ts index 2b47e730029..84d0a52d836 100644 --- a/apps/sim/lib/sim-search/live/text-match.ts +++ b/apps/sim/lib/sim-search/live/text-match.ts @@ -10,6 +10,12 @@ export type TextMatch = 'all' | 'any' /** Terms an any-term query sends; more would make the provider query long without adding recall. */ const MAX_ANY_TERMS = 8 +/** + * Terms a Jira any-term query pairs. Pairs grow quadratically, and a long OR of `text ~` pairs + * can run past the request timeout on a large site; six terms make at most 15 pairs. + */ +const MAX_JIRA_PAIR_TERMS = 6 + /** Words that match nearly every page, so they would make an any-term query return noise. */ const STOPWORDS = new Set([ 'about', @@ -57,15 +63,25 @@ const STOPWORDS = new Set([ * two means an any-term query would match the same pages as the all-terms one. */ export function significantTerms(query: string): string[] { + return distinctTerms(query, (term) => term.toLowerCase()).slice(0, MAX_ANY_TERMS) +} + +function distinctTerms(query: string, keyOf: (term: string) => string): string[] { const seen = new Set() - return queryTerms(query) - .filter((term) => { - const key = term.toLowerCase() - if (STOPWORDS.has(key) || seen.has(key)) return false - seen.add(key) - return true - }) - .slice(0, MAX_ANY_TERMS) + return queryTerms(query).filter((term) => { + const key = keyOf(term) + if (STOPWORDS.has(term.toLowerCase()) || seen.has(key)) return false + seen.add(key) + return true + }) +} + +/** One key for a word and its regular plural (`token`/`tokens`, `process`/`processes`). */ +function singularKey(term: string): string { + const key = term.toLowerCase() + if (key.length > 4 && key.endsWith('ies')) return `${key.slice(0, -3)}y` + const stem = key.endsWith('s') && !key.endsWith('ss') ? key.slice(0, -1) : key + return /(?:s|x|z|ch|sh)e$/.test(stem) ? stem.slice(0, -1) : stem } /** Escapes a value for a quoted JQL or CQL string, treating Lucene operators as literal text. */ @@ -73,11 +89,13 @@ const escapeSearchPhrase = (value: string) => JSON.stringify(value.replace(/[+\-&|!(){}[\]^"~*?:\\/]/g, '\\$&')) /** - * The `text ~` clause for a plain query in Jira JQL. `any` ORs every pair of terms, so a match - * contains at least two of them somewhere in the issue, or either term of a two-term query. + * The `text ~` clause for a plain query in Jira JQL. `any` ORs every pair of the query's longest + * terms, so a match contains at least two of them somewhere in the issue, or either term of a + * two-term query. Jira stems text search, so a word and its plural share one term. */ export function jiraTextClause(text: string, match: TextMatch = 'all'): string { - const terms = match === 'any' ? significantTerms(text) : [] + const terms = + match === 'any' ? distinctTerms(text, singularKey).slice(0, MAX_JIRA_PAIR_TERMS) : [] if (terms.length < 2) return `text ~ ${escapeSearchPhrase(text)}` const clause = (term: string) => `text ~ ${escapeSearchPhrase(term)}` const groups = diff --git a/apps/sim/lib/sim-search/live/types.ts b/apps/sim/lib/sim-search/live/types.ts index 24a06185e85..a9cb8baee12 100644 --- a/apps/sim/lib/sim-search/live/types.ts +++ b/apps/sim/lib/sim-search/live/types.ts @@ -82,6 +82,8 @@ export interface NativeClient { googleService?: 'sheets' /** Reuse this client's earlier response to the same GET instead of requesting it again. */ memo?: boolean + /** Cancels this request alongside the client's own signal. */ + signal?: AbortSignal } ): Promise text(path: string, query?: Record): Promise @@ -97,4 +99,9 @@ export interface NativeSearchInput { scopes: readonly string[] /** How a plain query's words must match; providers without full-text search ignore it. */ textMatch?: TextMatch + /** + * Cancels this search's provider requests alongside the client's own signal. Honored by the + * providers that run the any-term fallback, so its budget stops the request itself. + */ + signal?: AbortSignal } From 0b33d09e3b1b1cb38e1d01bc6f40c22baf2f6718 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 13 Oct 2026 13:22:37 -0700 Subject: [PATCH 2/3] improvement(search): keep Jira's eight-term pairing, simplify fallback error handling Drop the six-term pair cap (the budget already bounds a slow relaxed query, and the cap removed pairs of the shortest terms). Keep the singular/plural collapse, now leaving -us words (status, focus) whole. One branch per relaxed failure kind, the term cap inside distinctTerms, and no memo special case no caller reaches. --- apps/sim/lib/sim-search/live/http.ts | 2 +- apps/sim/lib/sim-search/live/term-fallback.ts | 32 ++++++++++--------- .../lib/sim-search/live/text-match.test.ts | 10 +++--- apps/sim/lib/sim-search/live/text-match.ts | 27 +++++++--------- 4 files changed, 33 insertions(+), 38 deletions(-) diff --git a/apps/sim/lib/sim-search/live/http.ts b/apps/sim/lib/sim-search/live/http.ts index 8d9ebf4fa1a..d2b7ea5ddb6 100644 --- a/apps/sim/lib/sim-search/live/http.ts +++ b/apps/sim/lib/sim-search/live/http.ts @@ -146,7 +146,7 @@ export function withJsonMemo(client: NativeClient): NativeClient { ...client, json(...request) { const [path, options] = request - if (!options?.memo || options.body || options.signal) return client.json(...request) + if (!options?.memo || options.body) return client.json(...request) const key = JSON.stringify([path, options.query, options.googleService]) let pending = memo.get(key) if (!pending) { diff --git a/apps/sim/lib/sim-search/live/term-fallback.ts b/apps/sim/lib/sim-search/live/term-fallback.ts index 7d323bc4f6e..b688e937a0d 100644 --- a/apps/sim/lib/sim-search/live/term-fallback.ts +++ b/apps/sim/lib/sim-search/live/term-fallback.ts @@ -16,9 +16,8 @@ const ANY_TERM_CANDIDATES = 25 const ANY_TERM_RESULTS = 5 /** - * Time the any-term search may take before the all-terms page is returned without it. It runs - * after the all-terms search inside the account's 12 s deadline and before the 2.5 s preview - * reads, and a provider's slow OR query must not cost the account the matches it already has. + * Time the any-term search may take before the all-terms page is returned without it, so a slow + * OR query cannot cost the account the matches it already has. */ const ANY_TERM_BUDGET_MS = 4000 @@ -71,21 +70,24 @@ export async function searchWithAnyTermFallback( ...input, limit: Math.max(input.limit, ANY_TERM_CANDIDATES), textMatch: 'any', - signal: AbortSignal.any([signal, budget.signal, ...(input.signal ? [input.signal] : [])]), + signal: AbortSignal.any([signal, budget.signal]), }) } catch (error) { signal.throwIfAborted() - input.signal?.throwIfAborted() - if (!(error instanceof NativeSearchError)) - logger.warn('Any-term search failed; keeping the all-terms page', { - timedOut: budget.signal.aborted, - error: getErrorMessage(error), - }) - const reason = - error instanceof NativeSearchError - ? error.message - : 'Results containing only some of the search terms could not be retrieved, so only results containing every term are shown.' - return { ...strict, partial: true, message: joinMessages([strict.message, reason]) } + if (error instanceof NativeSearchError) + return { ...strict, partial: true, message: joinMessages([strict.message, error.message]) } + logger.warn('Any-term search failed; keeping the all-terms page', { + timedOut: budget.signal.aborted, + error: getErrorMessage(error), + }) + return { + ...strict, + partial: true, + message: joinMessages([ + strict.message, + 'Results containing only some of the search terms could not be retrieved, so only results containing every term are shown.', + ]), + } } finally { clearTimeout(timer) } diff --git a/apps/sim/lib/sim-search/live/text-match.test.ts b/apps/sim/lib/sim-search/live/text-match.test.ts index 158a2fecb68..acf083fadea 100644 --- a/apps/sim/lib/sim-search/live/text-match.test.ts +++ b/apps/sim/lib/sim-search/live/text-match.test.ts @@ -18,16 +18,12 @@ describe('text match clauses', () => { expect(jiraTextClause('API api rollout', 'any')).toBe('(text ~ "rollout" OR text ~ "API")') }) - it('pairs only the six longest distinct terms of a long query, so the JQL stays bounded', () => { + it('pairs up to eight distinct terms of a long query', () => { const clause = jiraTextClause( 'rotate expired gateway credentials before quarterly compliance audit deadline', 'any' ) - expect(clause.match(/text ~/g)).toHaveLength(30) - for (const term of ['credentials', 'compliance', 'quarterly', 'deadline', 'expired', 'gateway']) - expect(clause).toContain(`"${term}"`) - expect(clause).not.toContain('"rotate"') - expect(clause).not.toContain('"audit"') + expect(clause.match(/text ~/g)).toHaveLength(56) }) it('spends one pair slot on a word and its plural, which Jira stems alike', () => { @@ -35,6 +31,8 @@ describe('text match clauses', () => { '(text ~ "rotation" OR text ~ "tokens")' ) expect(jiraTextClause('processes process', 'any')).toBe('text ~ "processes process"') + expect(jiraTextClause('statuses status', 'any')).toBe('text ~ "statuses status"') + expect(jiraTextClause('status focus', 'any')).toBe('(text ~ "status" OR text ~ "focus")') }) it('relaxes a Confluence query to its relevance-ranked site search', () => { diff --git a/apps/sim/lib/sim-search/live/text-match.ts b/apps/sim/lib/sim-search/live/text-match.ts index 84d0a52d836..7160078d6d3 100644 --- a/apps/sim/lib/sim-search/live/text-match.ts +++ b/apps/sim/lib/sim-search/live/text-match.ts @@ -10,12 +10,6 @@ export type TextMatch = 'all' | 'any' /** Terms an any-term query sends; more would make the provider query long without adding recall. */ const MAX_ANY_TERMS = 8 -/** - * Terms a Jira any-term query pairs. Pairs grow quadratically, and a long OR of `text ~` pairs - * can run past the request timeout on a large site; six terms make at most 15 pairs. - */ -const MAX_JIRA_PAIR_TERMS = 6 - /** Words that match nearly every page, so they would make an any-term query return noise. */ const STOPWORDS = new Set([ 'about', @@ -63,24 +57,26 @@ const STOPWORDS = new Set([ * two means an any-term query would match the same pages as the all-terms one. */ export function significantTerms(query: string): string[] { - return distinctTerms(query, (term) => term.toLowerCase()).slice(0, MAX_ANY_TERMS) + return distinctTerms(query, (term) => term.toLowerCase()) } function distinctTerms(query: string, keyOf: (term: string) => string): string[] { const seen = new Set() - return queryTerms(query).filter((term) => { - const key = keyOf(term) - if (STOPWORDS.has(term.toLowerCase()) || seen.has(key)) return false - seen.add(key) - return true - }) + return queryTerms(query) + .filter((term) => { + const key = keyOf(term) + if (STOPWORDS.has(term.toLowerCase()) || seen.has(key)) return false + seen.add(key) + return true + }) + .slice(0, MAX_ANY_TERMS) } /** One key for a word and its regular plural (`token`/`tokens`, `process`/`processes`). */ function singularKey(term: string): string { const key = term.toLowerCase() if (key.length > 4 && key.endsWith('ies')) return `${key.slice(0, -3)}y` - const stem = key.endsWith('s') && !key.endsWith('ss') ? key.slice(0, -1) : key + const stem = /[^su]s$/.test(key) ? key.slice(0, -1) : key return /(?:s|x|z|ch|sh)e$/.test(stem) ? stem.slice(0, -1) : stem } @@ -94,8 +90,7 @@ const escapeSearchPhrase = (value: string) => * two-term query. Jira stems text search, so a word and its plural share one term. */ export function jiraTextClause(text: string, match: TextMatch = 'all'): string { - const terms = - match === 'any' ? distinctTerms(text, singularKey).slice(0, MAX_JIRA_PAIR_TERMS) : [] + const terms = match === 'any' ? distinctTerms(text, singularKey) : [] if (terms.length < 2) return `text ~ ${escapeSearchPhrase(text)}` const clause = (term: string) => `text ~ ${escapeSearchPhrase(term)}` const groups = From 5393cb84c7555ab7a55afe74ff6978ce1eeea1b3 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 13 Oct 2026 13:32:32 -0700 Subject: [PATCH 3/3] fix(search): search an organization only through its integrations page connections Organization live search also listed a member's plain OAuth credentials at organization scope, which no page shows or can disconnect. One left behind next to the same person's integrations page connection made every result from that provider come back twice. Organization search now uses only credential-group enrollments; workspace search is unchanged. --- apps/sim/lib/sim-search/live/accounts.test.ts | 19 ++++++++++ apps/sim/lib/sim-search/live/accounts.ts | 37 +++++-------------- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/apps/sim/lib/sim-search/live/accounts.test.ts b/apps/sim/lib/sim-search/live/accounts.test.ts index 6f7d5f5f08e..249d3c7d698 100644 --- a/apps/sim/lib/sim-search/live/accounts.test.ts +++ b/apps/sim/lib/sim-search/live/accounts.test.ts @@ -130,6 +130,25 @@ describe('live account discovery boundaries', () => { }) expect(mocks.resolve).not.toHaveBeenCalled() }) + it('searches an organization only through its integrations page enrollments', async () => { + mockListOrganizationSearchApprovals.mockResolvedValue(new Map([['jira', true]])) + mocks.managed.mockResolvedValue([ + { id: 'enrolled', providerId: 'jira', displayName: 'Enrolled Jira' }, + ]) + const legacy = { id: 'legacy', providerId: 'jira', displayName: 'Legacy Jira' } + const metadata = ['enrolled', 'legacy'].map((id) => ({ + id, + scope: 'read:jira-work', + grantedScopes: null, + revokedAt: null, + })) + queueTableRows(schemaMock.credential, [legacy]) + queueTableRows(schemaMock.credential, metadata) + queueTableRows(schemaMock.credential, metadata) + expect( + (await listLiveAccounts({ organizationId: 'org' }, 'reader')).map((row) => row.id) + ).toEqual(['enrolled']) + }) it('does not resolve another account even when the caller knows its ID', async () => { metadata() await expect(resolveLiveAccount(owner, 'reader', 'someone-else')).rejects.toMatchObject({ diff --git a/apps/sim/lib/sim-search/live/accounts.ts b/apps/sim/lib/sim-search/live/accounts.ts index 7b333485674..1764878252e 100644 --- a/apps/sim/lib/sim-search/live/accounts.ts +++ b/apps/sim/lib/sim-search/live/accounts.ts @@ -1,6 +1,6 @@ import { db } from '@sim/db' import { account, credential, user } from '@sim/db/schema' -import { and, eq, inArray, isNull, ne } from 'drizzle-orm' +import { and, eq, inArray, isNull } from 'drizzle-orm' import { liveSearchProviderSchema } from '@/lib/api/contracts/mothership-assistant-tools' import { type ResourceOwner, @@ -28,34 +28,17 @@ import { } from '@/lib/sim-search/live/provider-catalog' import type { LiveAccount } from '@/lib/sim-search/live/types' +/** + * A workspace searches the person's workspace credentials; an organization searches only the + * accounts its members connect on the integrations page (credential-group enrollments). + */ async function listOAuthRows(scope: ResourceScope, userId: string) { if (scope.kind === 'workspace') return getPersonalOAuthCredentials(scope.workspaceId, userId) - const [managed, personal] = await Promise.all([ - getOwnOrganizationManagedOAuthCredentials({ organizationId: scope.organizationId, userId }), - db - .select({ - id: credential.id, - providerId: credential.providerId, - displayName: credential.displayName, - }) - .from(credential) - .innerJoin(account, eq(account.id, credential.accountId)) - .where( - and( - resourceScopeCondition(credential, scope), - eq(credential.type, 'oauth'), - eq(account.userId, userId), - eq(account.providerId, credential.providerId), - ne(credential.providerId, 'slack') - ) - ), - ]) - return [ - ...managed.map((row) => ({ ...row, type: 'managed_oauth' as const })), - ...personal.flatMap((row) => - row.providerId ? [{ ...row, providerId: row.providerId, type: 'oauth' as const }] : [] - ), - ] + const managed = await getOwnOrganizationManagedOAuthCredentials({ + organizationId: scope.organizationId, + userId, + }) + return managed.map((row) => ({ ...row, type: 'managed_oauth' as const })) } async function listCodaTokenRows(scope: ResourceScope, userId: string) {