diff --git a/packages/plugin-runtime/package.json b/packages/plugin-runtime/package.json index 778ee32..794f206 100644 --- a/packages/plugin-runtime/package.json +++ b/packages/plugin-runtime/package.json @@ -55,6 +55,7 @@ "domhandler": "^6.0.1", "env-paths": "^4.0.0", "htmlparser2": "^9.1.0", + "impit": "^0.14.9", "jszip": "^3.10.2", "protobufjs": "^7.6.6", "sanitize-html": "^2.17.7", diff --git a/packages/plugin-runtime/src/errors.ts b/packages/plugin-runtime/src/errors.ts index a33a6be..67410d0 100644 --- a/packages/plugin-runtime/src/errors.ts +++ b/packages/plugin-runtime/src/errors.ts @@ -1,3 +1,4 @@ +import impit from 'impit'; import { PluginError } from './plugins/errors.js'; export class HttpError extends Error { @@ -121,7 +122,7 @@ export function errorInfo(err: unknown): ErrorInfo { } } for (const e of chain(err)) { - if (e instanceof HttpError) + if (e instanceof HttpError || e instanceof impit.TransportError) return { code: 'NETWORK', message, diff --git a/packages/plugin-runtime/src/net/client.ts b/packages/plugin-runtime/src/net/client.ts index fa2c995..cb550aa 100644 --- a/packages/plugin-runtime/src/net/client.ts +++ b/packages/plugin-runtime/src/net/client.ts @@ -1,5 +1,6 @@ import type { CookieJar } from 'tough-cookie'; import { ChallengeError, HttpError } from '../errors.js'; +import { createImpitFetch, toHeaders } from './impit.js'; import { HostLimiter, type LimiterOptions } from './limiter.js'; export type FetchLike = ( @@ -26,12 +27,6 @@ export type HttpClientOptions = LimiterOptions & { sleep?: (ms: number, signal?: AbortSignal) => Promise; }; -const DEFAULT_HEADERS: Record = { - Accept: '*/*', - 'Accept-Language': 'en-US,en;q=0.9', - 'Sec-Fetch-Mode': 'cors', -}; - const REDIRECTS = new Set([301, 302, 303, 307, 308]); export { HttpError }; @@ -83,18 +78,6 @@ export async function isChallenge(res: Response): Promise { } } -type HeadersInit = ConstructorParameters[0]; - -function toHeaders(init: HeadersInit | undefined): Headers { - const headers = new Headers(); - if (init) { - const source = - init instanceof Headers ? init : new Headers(init as HeadersInit); - source.forEach((v, k) => headers.set(k, v)); - } - return headers; -} - /** * Polite HTTP client: per-host rate limiting, cookie jars that also see * redirect hops, and retries with exponential backoff honoring `Retry-After`. @@ -106,8 +89,7 @@ export class HttpClient { constructor(private readonly options: HttpClientOptions) { this.limiter = new HostLimiter(options); - this.fetchImpl = - options.fetch ?? ((input, init) => globalThis.fetch(input, init)); + this.fetchImpl = options.fetch ?? createImpitFetch(); this.sleep = options.sleep ?? sleep; } @@ -164,9 +146,6 @@ export class HttpClient { let method = (init.method ?? 'GET').toUpperCase(); let body = init.body; const baseHeaders = toHeaders(init.headers); - for (const [k, v] of Object.entries(DEFAULT_HEADERS)) { - if (!baseHeaders.has(k)) baseHeaders.set(k, v); - } if (!baseHeaders.has('user-agent')) baseHeaders.set('user-agent', opts.userAgent ?? this.options.userAgent); diff --git a/packages/plugin-runtime/src/net/impit.ts b/packages/plugin-runtime/src/net/impit.ts new file mode 100644 index 0000000..c3063be --- /dev/null +++ b/packages/plugin-runtime/src/net/impit.ts @@ -0,0 +1,99 @@ +import { Impit, type RequestInit as ImpitRequestInit } from 'impit'; +import { DEFAULT_USER_AGENT } from '../store/config.js'; +import type { FetchLike } from './client.js'; + +type HeadersInit = ConstructorParameters[0]; + +export function toHeaders(init: HeadersInit | undefined): Headers { + const headers = new Headers(); + if (init) { + const source = + init instanceof Headers ? init : new Headers(init as HeadersInit); + source.forEach((v, k) => headers.set(k, v)); + } + return headers; +} + +function secFetchSite(url: string, headers: Headers): string { + const source = headers.get('origin') ?? headers.get('referer'); + if (!source) return 'none'; + try { + return new URL(source).origin === new URL(url).origin + ? 'same-origin' + : 'cross-site'; + } catch { + return 'same-origin'; + } +} + +/** + * Adjusts only context-dependent headers (`Referer`, `Sec-Fetch-*`, and custom + * `User-Agent` overrides) while leaving static browser headers (`Accept`, + * `Accept-Encoding`, `Accept-Language`, `Priority`, `Sec-CH-UA*`, `User-Agent`) + * to impit's browser profile. + */ +function adaptHeaders( + url: string, + method: string, + init: HeadersInit | undefined, +): Headers { + const headers = toHeaders(init); + const ua = headers.get('user-agent'); + if (ua === DEFAULT_USER_AGENT) { + headers.delete('user-agent'); + } else if (ua) { + for (const h of ['sec-ch-ua', 'sec-ch-ua-mobile', 'sec-ch-ua-platform']) { + if (!headers.has(h)) headers.set(h, ''); + } + } + + if (!headers.has('referer')) { + try { + headers.set('referer', `${new URL(url).origin}/`); + } catch { + // Leave unset if URL parsing fails. + } + } + if (!headers.has('sec-fetch-site')) { + headers.set('sec-fetch-site', secFetchSite(url, headers)); + } + + const accept = headers.get('accept'); + const isImage = method === 'GET' && !!accept?.startsWith('image/'); + const isXhr = + !isImage && + (method !== 'GET' || + headers.has('x-requested-with') || + headers.has('origin') || + headers.has('content-type') || + headers.has('authorization') || + (accept !== null && !accept.includes('text/html'))); + + if (isImage) { + if (!headers.has('sec-fetch-mode')) + headers.set('sec-fetch-mode', 'no-cors'); + if (!headers.has('sec-fetch-dest')) headers.set('sec-fetch-dest', 'image'); + if (!headers.has('sec-fetch-user')) headers.set('sec-fetch-user', ''); + if (!headers.has('upgrade-insecure-requests')) + headers.set('upgrade-insecure-requests', ''); + } else if (isXhr) { + if (!headers.has('accept')) headers.set('accept', '*/*'); + if (!headers.has('sec-fetch-mode')) headers.set('sec-fetch-mode', 'cors'); + if (!headers.has('sec-fetch-dest')) headers.set('sec-fetch-dest', 'empty'); + if (!headers.has('sec-fetch-user')) headers.set('sec-fetch-user', ''); + if (!headers.has('upgrade-insecure-requests')) + headers.set('upgrade-insecure-requests', ''); + } + return headers; +} + +export function createImpitFetch(): FetchLike { + const impit = new Impit({ browser: 'chrome151' }); + return (input, init = {}) => { + const method = (init.method ?? 'GET').toUpperCase(); + return impit.fetch(input, { + ...(init as ImpitRequestInit), + headers: adaptHeaders(input, method, init.headers), + }) as unknown as Promise; + }; +} diff --git a/packages/plugin-runtime/src/store/config.ts b/packages/plugin-runtime/src/store/config.ts index 01645fd..a57e3f7 100644 --- a/packages/plugin-runtime/src/store/config.ts +++ b/packages/plugin-runtime/src/store/config.ts @@ -11,7 +11,7 @@ export const DEFAULT_REPO = 'https://raw.githubusercontent.com/LNReader/lnreader-plugins/plugins/v3.0.0/.dist/plugins.min.json'; export const DEFAULT_USER_AGENT = - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36'; + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36'; export const ConfigSchema = z.object({ repos: z.array(z.url()).default([DEFAULT_REPO]), diff --git a/packages/plugin-runtime/test/client.test.ts b/packages/plugin-runtime/test/client.test.ts index 39bb5b7..20205e9 100644 --- a/packages/plugin-runtime/test/client.test.ts +++ b/packages/plugin-runtime/test/client.test.ts @@ -1,6 +1,9 @@ +import { createServer } from 'node:http'; +import type { AddressInfo } from 'node:net'; import { CookieJar } from 'tough-cookie'; import { describe, expect, it } from 'vitest'; import { HttpClient, parseRetryAfter } from '../src/net/client.js'; +import { DEFAULT_USER_AGENT } from '../src/store/config.js'; import { mockFetch } from './helpers.js'; const client = ( @@ -101,4 +104,53 @@ describe('HttpClient', () => { }); expect(new Headers(calls[0]!.init.headers).get('user-agent')).toBe('mine'); }); + + it('emulates a browser with impit by default', async () => { + const server = createServer((req, res) => { + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify(req.headers)); + }); + await new Promise(r => server.listen(0, '127.0.0.1', r)); + const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}/`; + try { + const http = new HttpClient({ + concurrency: 1, + minGapMs: 0, + userAgent: DEFAULT_USER_AGENT, + }); + const defaultHeaders = (await (await http.request(url)).json()) as Record< + string, + string + >; + expect(defaultHeaders['user-agent']).toBe(DEFAULT_USER_AGENT); + expect(defaultHeaders['sec-ch-ua']).toContain('Google Chrome'); + expect(defaultHeaders['sec-fetch-mode']).toBe('navigate'); + + const customHeaders = (await ( + await http.request(url, {}, { userAgent: 'CustomBrowser/1.0' }) + ).json()) as Record; + expect(customHeaders['user-agent']).toBe('CustomBrowser/1.0'); + expect(customHeaders['sec-ch-ua']).toBeUndefined(); + + const imageHeaders = (await ( + await http.request(url, { + headers: { accept: 'image/avif,image/webp,image/*;q=0.8' }, + }) + ).json()) as Record; + expect(imageHeaders['sec-fetch-mode']).toBe('no-cors'); + expect(imageHeaders['sec-fetch-dest']).toBe('image'); + expect(imageHeaders['upgrade-insecure-requests']).toBeUndefined(); + + const jsonHeaders = (await ( + await http.request(url, { + headers: { accept: 'application/json' }, + }) + ).json()) as Record; + expect(jsonHeaders['sec-fetch-mode']).toBe('cors'); + expect(jsonHeaders['sec-fetch-dest']).toBe('empty'); + } finally { + server.closeAllConnections(); + await new Promise(r => server.close(() => r())); + } + }); }); diff --git a/packages/plugin-runtime/test/read.test.ts b/packages/plugin-runtime/test/read.test.ts index bb7f5e4..666320a 100644 --- a/packages/plugin-runtime/test/read.test.ts +++ b/packages/plugin-runtime/test/read.test.ts @@ -1,3 +1,4 @@ +import impit from 'impit'; import { describe, expect, it } from 'vitest'; import { BudgetExceededError, @@ -256,6 +257,10 @@ describe('errorInfo', () => { expect(errorInfo(new HttpError(500, 'https://x.test')).code).toBe( 'NETWORK', ); + expect( + errorInfo(new Error('failed', { cause: new impit.ConnectError('dns') })) + .code, + ).toBe('NETWORK'); expect(errorInfo(new PluginError('p', 'm', new Error('boom'))).code).toBe( 'PLUGIN_ERROR', ); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9f022a6..5f2ea70 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -99,6 +99,9 @@ importers: htmlparser2: specifier: ^9.1.0 version: 9.1.0 + impit: + specifier: ^0.14.9 + version: 0.14.9 jszip: specifier: ^3.10.2 version: 3.10.2 @@ -1500,6 +1503,62 @@ packages: immediate@3.0.6: resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==} + impit-darwin-arm64@0.14.9: + resolution: {integrity: sha512-jUFN+S3X9OzQgcVMHjrhCAizJtNwP1MbYUe2+Y48HFpwVlc4mW+9jBu0xMR68vQJg7l0cpeVC4pJTAtsw3laPQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + impit-darwin-x64@0.14.9: + resolution: {integrity: sha512-Qxj0jQl0DZWU0nkR+IDLf0kltJqvcvtCaqKiL6ZhKouGdbFHKZHjzBwL+jAQSjiIF8x7O3DKKNf3VyLraN78bQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + impit-linux-arm64-gnu@0.14.9: + resolution: {integrity: sha512-rcjpzOuweA+3FJsHAawM/driBtCyuxgyGIbO5zJ1KH62Qqnk9Z+B5UrBBJqsLdgpTrZ6ueZ2SuBsWB8iMXw84w==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + impit-linux-arm64-musl@0.14.9: + resolution: {integrity: sha512-EdQDSuyM28USb6WDH16k/UMQbBQI/MkEB5QcJ6YKaC7SemwMIZZBNpAE+fnEV6uPEcZl5ZmAL4cC1sb0Dy+Rcg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [musl] + + impit-linux-x64-gnu@0.14.9: + resolution: {integrity: sha512-G8yPPToii/Clbl9cJ8EtSZ/iXBV9O9SDbT+Cn9L1etLle8T+47vq45iZ++M8t8AXnzmP0S+Vmojd3y8Co+IrEQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [glibc] + + impit-linux-x64-musl@0.14.9: + resolution: {integrity: sha512-Uk3OF+Up2WFk1E2xupkXNVyYrK1E/XwDD1RNX40kQuOTOiFKVb0yT+R5okNmRzQEYvY6q4vO3Xyoy92Tc71sAg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [musl] + + impit-win32-arm64-msvc@0.14.9: + resolution: {integrity: sha512-revggTn2zIGPv61XIM9Qc7NHDtU8R2I7E0YsbjUhb8J4kACTezpN+326BxoOud7in9WgKQheB2tBw3d4w6KYhg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + impit-win32-x64-msvc@0.14.9: + resolution: {integrity: sha512-Ob1My2vzT64i1/1GlIp7cIXQ74ia8x3z56IYPmq6B2ltnBmSqjFh9lcOliSi7SWvhXlpaB5uKz6DphKj27nQ4Q==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + impit@0.14.9: + resolution: {integrity: sha512-vXnHi+8Tih1amM0GS3gu+tJS/H9U179OgKxSJEybK+pYB8T3pJGHstp1OWtFcM5qc/vGzClH4npVs1NkIsF98w==} + engines: {node: '>= 20'} + import-meta-resolve@4.2.0: resolution: {integrity: sha512-Iqv2fzaTQN28s/FwZAoFq0ZSs/7hMAHJVX+w8PZl3cY19Pxk6jFFalxQoIfW2826i/fDLXv8IiEZRIT0lDuWcg==} @@ -3504,6 +3563,41 @@ snapshots: immediate@3.0.6: {} + impit-darwin-arm64@0.14.9: + optional: true + + impit-darwin-x64@0.14.9: + optional: true + + impit-linux-arm64-gnu@0.14.9: + optional: true + + impit-linux-arm64-musl@0.14.9: + optional: true + + impit-linux-x64-gnu@0.14.9: + optional: true + + impit-linux-x64-musl@0.14.9: + optional: true + + impit-win32-arm64-msvc@0.14.9: + optional: true + + impit-win32-x64-msvc@0.14.9: + optional: true + + impit@0.14.9: + optionalDependencies: + impit-darwin-arm64: 0.14.9 + impit-darwin-x64: 0.14.9 + impit-linux-arm64-gnu: 0.14.9 + impit-linux-arm64-musl: 0.14.9 + impit-linux-x64-gnu: 0.14.9 + impit-linux-x64-musl: 0.14.9 + impit-win32-arm64-msvc: 0.14.9 + impit-win32-x64-msvc: 0.14.9 + import-meta-resolve@4.2.0: {} imurmurhash@0.1.4: {} diff --git a/scripts/pack-check.mjs b/scripts/pack-check.mjs index e0b09fe..03f16ff 100644 --- a/scripts/pack-check.mjs +++ b/scripts/pack-check.mjs @@ -217,9 +217,9 @@ try { } catch { // Not JSON; reported below. } - // Plain connection failures (no HTTP response) surface as `fetch failed`. + // Plain connection failures (no HTTP response) surface as `fetch failed` or impit's `Failed to connect`. const offline = - /fetch failed|ENOTFOUND|EAI_AGAIN|ECONNREFUSED|ECONNRESET|ETIMEDOUT|UND_ERR|network/i; + /fetch failed|Failed to connect|ENOTFOUND|EAI_AGAIN|ECONNREFUSED|ECONNRESET|ETIMEDOUT|UND_ERR|network/i; if ( error.code === 'NETWORK' || offline.test(`${error.message ?? ''}\n${list.stderr}`)