From 6964753e3e60509884a9f33e2e06d2a07d571aaf Mon Sep 17 00:00:00 2001 From: Alvin-Zilverstand <524715@vistacollege.nl> Date: Mon, 21 Sep 2026 15:37:24 +0200 Subject: [PATCH] f --- README.md | 13 ++++ backend/twitter.go | 4 +- frontend/package.json | 1 + frontend/scripts/fetch-delay.test.mjs | 84 ++++++++++++++++++++++++ frontend/src/App.tsx | 6 ++ frontend/src/components/SettingsPage.tsx | 24 +++++++ frontend/src/lib/fetch-delay.ts | 27 ++++++++ frontend/src/lib/settings.ts | 7 ++ 8 files changed, 164 insertions(+), 2 deletions(-) create mode 100644 frontend/scripts/fetch-delay.test.mjs create mode 100644 frontend/src/lib/fetch-delay.ts diff --git a/README.md b/README.md index 544f608..fa0b3e0 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,19 @@ - Also available on the web at [**TwitterDL.app**](https://twitterdl.app) +## Fetch pacing + +Settings includes separate **Fetch Delay** controls: a base delay and a random +extra from zero to the configured value, in seconds. Both default to zero +(disabled). A suggested starting point is 5 seconds base plus up to 5 seconds +extra; this is not a guaranteed safe rate for Twitter/X. + +The delay spaces out extractor starts across batches, accounts, resumes, retries, +and date-range fetches. The first fetch starts immediately. It does not control +requests made internally by the external extractor. Stop interrupts a pending +wait, and waiting counts toward Fetch Timeout, so increase the timeout when +using longer delays. + ## How to Obtain Auth Token > [!Warning] diff --git a/backend/twitter.go b/backend/twitter.go index 9ec71d2..1ae62ef 100644 --- a/backend/twitter.go +++ b/backend/twitter.go @@ -730,7 +730,7 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) { args = append(args, "--guest") } - args = append(args, "--json", "--metadata") + args = append(args, "--json", "--metadata", "--no-discord") if req.BatchSize > 0 { args = append(args, "--limit", fmt.Sprintf("%d", req.BatchSize)) @@ -840,7 +840,7 @@ func ExtractDateRange(req DateRangeRequest) (*TwitterResponse, error) { args = append(args, "--guest") } - args = append(args, "--json", "--metadata") + args = append(args, "--json", "--metadata", "--no-discord") if req.Retweets { args = append(args, "--retweets", "include") diff --git a/frontend/package.json b/frontend/package.json index 6d3a912..87ff553 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -4,6 +4,7 @@ "version": "0.0.0", "type": "module", "scripts": { + "test": "node --experimental-strip-types --test scripts/*.test.mjs", "dev": "vite", "build": "tsc -b && vite build", "lint": "eslint .", diff --git a/frontend/scripts/fetch-delay.test.mjs b/frontend/scripts/fetch-delay.test.mjs new file mode 100644 index 0000000..aa4caa4 --- /dev/null +++ b/frontend/scripts/fetch-delay.test.mjs @@ -0,0 +1,84 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { createFetchDelayGate, normalizeFetchDelay } from "../src/lib/fetch-delay.ts"; + +function clock(random = 0.5) { + let time = 0; + const waits = []; + const gate = createFetchDelayGate({ + now: () => time, + random: () => random, + sleep: async (ms) => { waits.push(ms); time += ms; }, + }); + return { gate, waits, now: () => time, advance: (ms) => { time += ms; } }; +} + +test("old or invalid settings disable delay; valid settings are bounded", () => { + for (const value of [undefined, null, "5", NaN, Infinity, -1]) { + assert.equal(normalizeFetchDelay(value), 0); + } + assert.equal(normalizeFetchDelay(4000), 3600); + assert.equal(normalizeFetchDelay(1.2345), 1.235); +}); + +test("first fetch starts immediately; later batches/accounts share pacing", async () => { + const c = clock(); + assert.equal(await c.gate(5, 5, () => false), true); + assert.equal(c.now(), 0); + c.advance(2000); // Time spent fetching the previous batch. + assert.equal(await c.gate(5, 5, () => false), true); + assert.equal(c.now(), 7500); + await c.gate(5, 5, () => false); + assert.equal(c.now(), 15000); + assert.ok(c.waits.every((ms) => ms <= 100)); +}); + +test("disabled pacing and a long fetch do not introduce unnecessary waits", async () => { + const c = clock(); + await c.gate(0, 0, () => false); + await c.gate(0, 0, () => false); + c.advance(10000); + await c.gate(5, 0, () => false); + assert.deepEqual(c.waits, []); +}); + +test("random-only pacing supports both ends of the random range", async () => { + for (const random of [0, 0.999]) { + const c = clock(random); + await c.gate(0, 5, () => false); + await c.gate(0, 5, () => false); + assert.equal(c.now(), random * 5000); + } +}); + +test("stop or timeout during a wait prevents the next fetch from starting", async () => { + const c = clock(); + await c.gate(5, 0, () => false); + assert.equal(await c.gate(5, 0, () => c.now() >= 200), false); + assert.equal(c.now(), 200); + assert.equal(await c.gate(0, 0, () => true), false); + await c.gate(5, 0, () => false); + assert.equal(c.now(), 5000); // Cancellation did not reserve a future slot. +}); + +test("concurrent callers recheck the shared start time after waking", async () => { + let time = 0; + const sleepers = []; + const gate = createFetchDelayGate({ + now: () => time, + random: () => 0, + sleep: () => new Promise((resolve) => sleepers.push(resolve)), + }); + await gate(1, 0, () => false); + const starts = []; + const a = gate(1, 0, () => false).then(() => starts.push(time)); + const b = gate(1, 0, () => false).then(() => starts.push(time)); + time = 1000; + sleepers.splice(0).forEach((resolve) => resolve()); + await a; + assert.deepEqual(starts, [1000]); + time = 2000; + sleepers.splice(0).forEach((resolve) => resolve()); + await b; + assert.deepEqual(starts, [1000, 2000]); +}); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index c383c33..4c0e62b 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -3,6 +3,7 @@ import { TooltipProvider } from "@/components/ui/tooltip"; import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; import { Button } from "@/components/ui/button"; import { getSettings, applyThemeMode, applyFont } from "@/lib/settings"; +import { createFetchDelayGate } from "@/lib/fetch-delay"; import { applyTheme } from "@/lib/themes"; import { openExternal } from "@/lib/utils"; import { isNewerVersion } from "@/lib/version"; @@ -29,6 +30,7 @@ const HISTORY_KEY = "twitter_media_fetch_history"; const MAX_HISTORY = 10; const CURRENT_VERSION = __APP_VERSION__; const BATCH_SIZE = 100; +const waitForFetch = createFetchDelayGate(); function formatNumberWithComma(num: number): string { return num.toLocaleString(); } @@ -446,6 +448,7 @@ function App() { return; } } + if (!await waitForFetch(settings.fetchDelaySeconds, settings.fetchDelayJitterSeconds, () => stopFetchRef.current)) return; const response = await ExtractDateRange({ username: username.trim(), auth_token: authToken.trim(), @@ -522,6 +525,7 @@ function App() { } } const batchNum = page + 1; + if (!await waitForFetch(settings.fetchDelaySeconds, settings.fetchDelayJitterSeconds, () => stopFetchRef.current)) break; logger.info(`Fetching batch ${batchNum}${cursor ? " (resuming)" : ""}...`); const response = await ExtractTimeline({ username: isBookmarks ? "" : username.trim(), @@ -977,6 +981,7 @@ function App() { } break; } + if (!await waitForFetch(settings.fetchDelaySeconds, settings.fetchDelayJitterSeconds, () => stopAllRef.current || !!accountStopFlagsRef.current.get(accountId))) break; const response = await ExtractTimeline({ username: cleanUsername, auth_token: authToken.trim(), @@ -1336,6 +1341,7 @@ function App() { if (accountStopFlagsRef.current.get(accountId)) { break; } + if (!await waitForFetch(settings.fetchDelaySeconds, settings.fetchDelayJitterSeconds, () => stopAllRef.current || !!accountStopFlagsRef.current.get(accountId))) break; const response = await ExtractTimeline({ username: cleanUsername, auth_token: authToken.trim(), diff --git a/frontend/src/components/SettingsPage.tsx b/frontend/src/components/SettingsPage.tsx index 94b752a..d57e24d 100644 --- a/frontend/src/components/SettingsPage.tsx +++ b/frontend/src/components/SettingsPage.tsx @@ -743,6 +743,30 @@ export function SettingsPage({ onUnsavedChangesChange, onResetRequest }: Setting setTempSettings((prev) => ({ ...prev, proxy: e.target.value }))} placeholder="http://proxy1:port, socks5://proxy2:port (optional)" className="max-w-md"/> +
+ +
+ {([ + ["fetchDelaySeconds", "Base Delay"], + ["fetchDelayJitterSeconds", "Random Extra"], + ] as const).map(([key, label]) => ( +
+ +
+ { + const value = Number(e.target.value); + setTempSettings((prev) => ({ ...prev, [key]: Number.isFinite(value) ? Math.min(3600, Math.max(0, value)) : 0 })); + }} className="w-24"/> + sec +
+
+ ))} +
+

Spaces out fetch batch starts, including account switches and retries. Random Extra adds between 0 and the chosen number of seconds. Set both to 0 to disable.

+

Suggested starting point: 5 sec base + 5 sec random extra. No delay guarantees avoidance of rate limits. Waiting counts toward Fetch Timeout; increase it for longer delays.

+

Applies between extractor runs, including date-range fetches, rather than individual requests inside the extractor.

+
+