Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/plugin-runtime/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
"domhandler": "^6.0.1",
"env-paths": "^4.0.0",
"htmlparser2": "^9.1.0",
"impit": "^0.14.9",
"jszip": "^3.10.2",
"protobufjs": "^7.6.6",
"sanitize-html": "^2.17.7",
Expand Down
3 changes: 2 additions & 1 deletion packages/plugin-runtime/src/errors.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import impit from 'impit';

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This loads the native module whenever errors.ts is imported, and nearly everything imports it. Comparing by name avoids that, e.g. e instanceof Error && /^(TransportError|ConnectError|…)$/.test(e.name), or a small isImpitTransportError helper in net/impit.ts that doesn't need the module loaded.


Generated by Claude Code

import { PluginError } from './plugins/errors.js';

export class HttpError extends Error {
Expand Down Expand Up @@ -121,7 +122,7 @@ export function errorInfo(err: unknown): ErrorInfo {
}
}
for (const e of chain(err)) {
if (e instanceof HttpError)
if (e instanceof HttpError || e instanceof impit.TransportError)
return {
code: 'NETWORK',
message,
Expand Down
25 changes: 2 additions & 23 deletions packages/plugin-runtime/src/net/client.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import type { CookieJar } from 'tough-cookie';
import { ChallengeError, HttpError } from '../errors.js';
import { createImpitFetch, toHeaders } from './impit.js';
import { HostLimiter, type LimiterOptions } from './limiter.js';

export type FetchLike = (
Expand All @@ -26,12 +27,6 @@ export type HttpClientOptions = LimiterOptions & {
sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
};

const DEFAULT_HEADERS: Record<string, string> = {
Accept: '*/*',
'Accept-Language': 'en-US,en;q=0.9',
'Sec-Fetch-Mode': 'cors',
};

const REDIRECTS = new Set([301, 302, 303, 307, 308]);

export { HttpError };
Expand Down Expand Up @@ -83,18 +78,6 @@ export async function isChallenge(res: Response): Promise<boolean> {
}
}

type HeadersInit = ConstructorParameters<typeof Headers>[0];

function toHeaders(init: HeadersInit | undefined): Headers {
const headers = new Headers();
if (init) {
const source =
init instanceof Headers ? init : new Headers(init as HeadersInit);
source.forEach((v, k) => headers.set(k, v));
}
return headers;
}

/**
* Polite HTTP client: per-host rate limiting, cookie jars that also see
* redirect hops, and retries with exponential backoff honoring `Retry-After`.
Expand All @@ -106,8 +89,7 @@ export class HttpClient {

constructor(private readonly options: HttpClientOptions) {
this.limiter = new HostLimiter(options);
this.fetchImpl =
options.fetch ?? ((input, init) => globalThis.fetch(input, init));
this.fetchImpl = options.fetch ?? createImpitFetch();
this.sleep = options.sleep ?? sleep;
}

Expand Down Expand Up @@ -164,9 +146,6 @@ export class HttpClient {
let method = (init.method ?? 'GET').toUpperCase();
let body = init.body;
const baseHeaders = toHeaders(init.headers);
for (const [k, v] of Object.entries(DEFAULT_HEADERS)) {
if (!baseHeaders.has(k)) baseHeaders.set(k, v);
}
if (!baseHeaders.has('user-agent'))
baseHeaders.set('user-agent', opts.userAgent ?? this.options.userAgent);

Expand Down
99 changes: 99 additions & 0 deletions packages/plugin-runtime/src/net/impit.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
import { Impit, type RequestInit as ImpitRequestInit } from 'impit';

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

impit is a native module with prebuilt binaries for only 8 targets: macOS, Windows, and Linux glibc/musl, each on x64 and arm64. On any other platform (Termux/Android, FreeBSD, 32-bit ARM), this top-level import makes the whole CLI crash at startup. I checked by hiding the Linux binary, and even lnreader --help failed with impit's native binding error. Could you load it lazily inside createImpitFetch (await import('impit')) and fall back to globalThis.fetch if the import throws?


Generated by Claude Code

import { DEFAULT_USER_AGENT } from '../store/config.js';
import type { FetchLike } from './client.js';

type HeadersInit = ConstructorParameters<typeof Headers>[0];

export function toHeaders(init: HeadersInit | undefined): Headers {
const headers = new Headers();
if (init) {
const source =
init instanceof Headers ? init : new Headers(init as HeadersInit);
source.forEach((v, k) => headers.set(k, v));
}
return headers;
}

function secFetchSite(url: string, headers: Headers): string {
const source = headers.get('origin') ?? headers.get('referer');
if (!source) return 'none';
try {
return new URL(source).origin === new URL(url).origin
? 'same-origin'
: 'cross-site';
} catch {
return 'same-origin';
}
}

/**
* Adjusts only context-dependent headers (`Referer`, `Sec-Fetch-*`, and custom
* `User-Agent` overrides) while leaving static browser headers (`Accept`,
* `Accept-Encoding`, `Accept-Language`, `Priority`, `Sec-CH-UA*`, `User-Agent`)
* to impit's browser profile.
*/
function adaptHeaders(
url: string,
method: string,
init: HeadersInit | undefined,
): Headers {
const headers = toHeaders(init);
const ua = headers.get('user-agent');
if (ua === DEFAULT_USER_AGENT) {
headers.delete('user-agent');
} else if (ua) {
for (const h of ['sec-ch-ua', 'sec-ch-ua-mobile', 'sec-ch-ua-platform']) {
if (!headers.has(h)) headers.set(h, '');
}
}

if (!headers.has('referer')) {
try {
headers.set('referer', `${new URL(url).origin}/`);

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because Referer defaults to the target's own origin before secFetchSite runs, Sec-Fetch-Site is nearly always same-origin. The 'none' branch (line 19) never runs, and calls to other hosts send a self-referer. That's also recomputed on each redirect hop. Maybe work out Sec-Fetch-Site first and only add Referer when the caller didn't supply one?


Generated by Claude Code

} catch {
// Leave unset if URL parsing fails.
}
}
if (!headers.has('sec-fetch-site')) {
headers.set('sec-fetch-site', secFetchSite(url, headers));
}

const accept = headers.get('accept');
const isImage = method === 'GET' && !!accept?.startsWith('image/');
const isXhr =
!isImage &&
(method !== 'GET' ||
headers.has('x-requested-with') ||
headers.has('origin') ||
headers.has('content-type') ||
headers.has('authorization') ||
(accept !== null && !accept.includes('text/html')));

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: A GET with no Accept header falls through to navigation headers (text/html, Sec-Fetch-Mode: navigate). Before this PR it sent */* with cors. That's fine for pages, but plugins calling JSON APIs without setting Accept will now look like page loads. It's worth keeping in mind if any API plugins start acting differently.


Generated by Claude Code


if (isImage) {
if (!headers.has('sec-fetch-mode'))
headers.set('sec-fetch-mode', 'no-cors');
if (!headers.has('sec-fetch-dest')) headers.set('sec-fetch-dest', 'image');
if (!headers.has('sec-fetch-user')) headers.set('sec-fetch-user', '');
if (!headers.has('upgrade-insecure-requests'))
headers.set('upgrade-insecure-requests', '');
} else if (isXhr) {
if (!headers.has('accept')) headers.set('accept', '*/*');
if (!headers.has('sec-fetch-mode')) headers.set('sec-fetch-mode', 'cors');
if (!headers.has('sec-fetch-dest')) headers.set('sec-fetch-dest', 'empty');
if (!headers.has('sec-fetch-user')) headers.set('sec-fetch-user', '');
if (!headers.has('upgrade-insecure-requests'))
headers.set('upgrade-insecure-requests', '');
}
return headers;
}

export function createImpitFetch(): FetchLike {
const impit = new Impit({ browser: 'chrome151' });

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

impit has a default 30 s timeout that covers the whole request, body included. Undici had no overall limit. A slow server that sends headers and then streams for more than 30 s now fails during res.text()/arrayBuffer(). That error is a plain Error rather than a TransportError, so it isn't retried or reported as NETWORK. Suggest new Impit({ browser: 'chrome151', timeout: null }), or a deliberately chosen value.


Generated by Claude Code

return (input, init = {}) => {
const method = (init.method ?? 'GET').toUpperCase();
return impit.fetch(input, {
...(init as ImpitRequestInit),
headers: adaptHeaders(input, method, init.headers),
}) as unknown as Promise<Response>;
};
}
2 changes: 1 addition & 1 deletion packages/plugin-runtime/src/store/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ export const DEFAULT_REPO =
'https://raw.githubusercontent.com/LNReader/lnreader-plugins/plugins/v3.0.0/.dist/plugins.min.json';

export const DEFAULT_USER_AGENT =
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36';
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36';

export const ConfigSchema = z.object({
repos: z.array(z.url()).default([DEFAULT_REPO]),
Expand Down
52 changes: 52 additions & 0 deletions packages/plugin-runtime/test/client.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
import { createServer } from 'node:http';
import type { AddressInfo } from 'node:net';
import { CookieJar } from 'tough-cookie';
import { describe, expect, it } from 'vitest';
import { HttpClient, parseRetryAfter } from '../src/net/client.js';
import { DEFAULT_USER_AGENT } from '../src/store/config.js';
import { mockFetch } from './helpers.js';

const client = (
Expand Down Expand Up @@ -101,4 +104,53 @@ describe('HttpClient', () => {
});
expect(new Headers(calls[0]!.init.headers).get('user-agent')).toBe('mine');
});

it('emulates a browser with impit by default', async () => {
const server = createServer((req, res) => {
res.writeHead(200, { 'content-type': 'application/json' });
res.end(JSON.stringify(req.headers));
});
await new Promise<void>(r => server.listen(0, '127.0.0.1', r));
const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}/`;
try {
const http = new HttpClient({
concurrency: 1,
minGapMs: 0,
userAgent: DEFAULT_USER_AGENT,
});
const defaultHeaders = (await (await http.request(url)).json()) as Record<
string,
string
>;
expect(defaultHeaders['user-agent']).toBe(DEFAULT_USER_AGENT);
expect(defaultHeaders['sec-ch-ua']).toContain('Google Chrome');
expect(defaultHeaders['sec-fetch-mode']).toBe('navigate');

const customHeaders = (await (
await http.request(url, {}, { userAgent: 'CustomBrowser/1.0' })
).json()) as Record<string, string>;
expect(customHeaders['user-agent']).toBe('CustomBrowser/1.0');
expect(customHeaders['sec-ch-ua']).toBeUndefined();

const imageHeaders = (await (
await http.request(url, {
headers: { accept: 'image/avif,image/webp,image/*;q=0.8' },
})
).json()) as Record<string, string>;
expect(imageHeaders['sec-fetch-mode']).toBe('no-cors');
expect(imageHeaders['sec-fetch-dest']).toBe('image');
expect(imageHeaders['upgrade-insecure-requests']).toBeUndefined();

const jsonHeaders = (await (
await http.request(url, {
headers: { accept: 'application/json' },
})
).json()) as Record<string, string>;
expect(jsonHeaders['sec-fetch-mode']).toBe('cors');
expect(jsonHeaders['sec-fetch-dest']).toBe('empty');
} finally {
server.closeAllConnections();
await new Promise<void>(r => server.close(() => r()));
}
});
});
5 changes: 5 additions & 0 deletions packages/plugin-runtime/test/read.test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import impit from 'impit';
import { describe, expect, it } from 'vitest';
import {
BudgetExceededError,
Expand Down Expand Up @@ -256,6 +257,10 @@ describe('errorInfo', () => {
expect(errorInfo(new HttpError(500, 'https://x.test')).code).toBe(
'NETWORK',
);
expect(
errorInfo(new Error('failed', { cause: new impit.ConnectError('dns') }))
.code,
).toBe('NETWORK');
expect(errorInfo(new PluginError('p', 'm', new Error('boom'))).code).toBe(
'PLUGIN_ERROR',
);
Expand Down
94 changes: 94 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions scripts/pack-check.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -217,9 +217,9 @@ try {
} catch {
// Not JSON; reported below.
}
// Plain connection failures (no HTTP response) surface as `fetch failed`.
// Plain connection failures (no HTTP response) surface as `fetch failed` or impit's `Failed to connect`.
const offline =
/fetch failed|ENOTFOUND|EAI_AGAIN|ECONNREFUSED|ECONNRESET|ETIMEDOUT|UND_ERR|network/i;
/fetch failed|Failed to connect|ENOTFOUND|EAI_AGAIN|ECONNREFUSED|ECONNRESET|ETIMEDOUT|UND_ERR|network/i;
if (
error.code === 'NETWORK' ||
offline.test(`${error.message ?? ''}\n${list.stderr}`)
Expand Down
Loading