This commit is contained in:
Alvin-Zilverstand committed 2026-09-21 15:37:24 +02:00
1 parent 9c269bc8f2
commit 6964753e3e
8 files changed
+164 -2

No files matched your search

+13
View File
@@ -34,6 +34,19 @@
- Also available on the web at [**TwitterDL.app**](https://twitterdl.app)
## Fetch pacing
Settings includes separate **Fetch Delay** controls: a base delay and a random
extra from zero to the configured value, in seconds. Both default to zero
(disabled). A suggested starting point is 5 seconds base plus up to 5 seconds
extra; this is not a guaranteed safe rate for Twitter/X.
The delay spaces out extractor starts across batches, accounts, resumes, retries,
and date-range fetches. The first fetch starts immediately. It does not control
requests made internally by the external extractor. Stop interrupts a pending
wait, and waiting counts toward Fetch Timeout, so increase the timeout when
using longer delays.
## How to Obtain Auth Token
> [!Warning]
+2 -2
View File
@@ -730,7 +730,7 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) {
args = append(args, "--guest")
}
args = append(args, "--json", "--metadata")
args = append(args, "--json", "--metadata", "--no-discord")
if req.BatchSize > 0 {
args = append(args, "--limit", fmt.Sprintf("%d", req.BatchSize))
@@ -840,7 +840,7 @@ func ExtractDateRange(req DateRangeRequest) (*TwitterResponse, error) {
args = append(args, "--guest")
}
args = append(args, "--json", "--metadata")
args = append(args, "--json", "--metadata", "--no-discord")
if req.Retweets {
args = append(args, "--retweets", "include")
+1
View File
@@ -4,6 +4,7 @@
"version": "0.0.0",
"type": "module",
"scripts": {
"test": "node --experimental-strip-types --test scripts/*.test.mjs",
"dev": "vite",
"build": "tsc -b && vite build",
"lint": "eslint .",
+84
View File
@@ -0,0 +1,84 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { createFetchDelayGate, normalizeFetchDelay } from "../src/lib/fetch-delay.ts";
function clock(random = 0.5) {
let time = 0;
const waits = [];
const gate = createFetchDelayGate({
now: () => time,
random: () => random,
sleep: async (ms) => { waits.push(ms); time += ms; },
});
return { gate, waits, now: () => time, advance: (ms) => { time += ms; } };
}
test("old or invalid settings disable delay; valid settings are bounded", () => {
for (const value of [undefined, null, "5", NaN, Infinity, -1]) {
assert.equal(normalizeFetchDelay(value), 0);
}
assert.equal(normalizeFetchDelay(4000), 3600);
assert.equal(normalizeFetchDelay(1.2345), 1.235);
});
test("first fetch starts immediately; later batches/accounts share pacing", async () => {
const c = clock();
assert.equal(await c.gate(5, 5, () => false), true);
assert.equal(c.now(), 0);
c.advance(2000); // Time spent fetching the previous batch.
assert.equal(await c.gate(5, 5, () => false), true);
assert.equal(c.now(), 7500);
await c.gate(5, 5, () => false);
assert.equal(c.now(), 15000);
assert.ok(c.waits.every((ms) => ms <= 100));
});
test("disabled pacing and a long fetch do not introduce unnecessary waits", async () => {
const c = clock();
await c.gate(0, 0, () => false);
await c.gate(0, 0, () => false);
c.advance(10000);
await c.gate(5, 0, () => false);
assert.deepEqual(c.waits, []);
});
test("random-only pacing supports both ends of the random range", async () => {
for (const random of [0, 0.999]) {
const c = clock(random);
await c.gate(0, 5, () => false);
await c.gate(0, 5, () => false);
assert.equal(c.now(), random * 5000);
}
});
test("stop or timeout during a wait prevents the next fetch from starting", async () => {
const c = clock();
await c.gate(5, 0, () => false);
assert.equal(await c.gate(5, 0, () => c.now() >= 200), false);
assert.equal(c.now(), 200);
assert.equal(await c.gate(0, 0, () => true), false);
await c.gate(5, 0, () => false);
assert.equal(c.now(), 5000); // Cancellation did not reserve a future slot.
});
test("concurrent callers recheck the shared start time after waking", async () => {
let time = 0;
const sleepers = [];
const gate = createFetchDelayGate({
now: () => time,
random: () => 0,
sleep: () => new Promise((resolve) => sleepers.push(resolve)),
});
await gate(1, 0, () => false);
const starts = [];
const a = gate(1, 0, () => false).then(() => starts.push(time));
const b = gate(1, 0, () => false).then(() => starts.push(time));
time = 1000;
sleepers.splice(0).forEach((resolve) => resolve());
await a;
assert.deepEqual(starts, [1000]);
time = 2000;
sleepers.splice(0).forEach((resolve) => resolve());
await b;
assert.deepEqual(starts, [1000, 2000]);
});
+6
View File
@@ -3,6 +3,7 @@ import { TooltipProvider } from "@/components/ui/tooltip";
import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog";
import { Button } from "@/components/ui/button";
import { getSettings, applyThemeMode, applyFont } from "@/lib/settings";
import { createFetchDelayGate } from "@/lib/fetch-delay";
import { applyTheme } from "@/lib/themes";
import { openExternal } from "@/lib/utils";
import { isNewerVersion } from "@/lib/version";
@@ -29,6 +30,7 @@ const HISTORY_KEY = "twitter_media_fetch_history";
const MAX_HISTORY = 10;
const CURRENT_VERSION = __APP_VERSION__;
const BATCH_SIZE = 100;
const waitForFetch = createFetchDelayGate();
function formatNumberWithComma(num: number): string {
return num.toLocaleString();
}
@@ -446,6 +448,7 @@ function App() {
return;
}
}
if (!await waitForFetch(settings.fetchDelaySeconds, settings.fetchDelayJitterSeconds, () => stopFetchRef.current)) return;
const response = await ExtractDateRange({
username: username.trim(),
auth_token: authToken.trim(),
@@ -522,6 +525,7 @@ function App() {
}
}
const batchNum = page + 1;
if (!await waitForFetch(settings.fetchDelaySeconds, settings.fetchDelayJitterSeconds, () => stopFetchRef.current)) break;
logger.info(`Fetching batch ${batchNum}${cursor ? " (resuming)" : ""}...`);
const response = await ExtractTimeline({
username: isBookmarks ? "" : username.trim(),
@@ -977,6 +981,7 @@ function App() {
}
break;
}
if (!await waitForFetch(settings.fetchDelaySeconds, settings.fetchDelayJitterSeconds, () => stopAllRef.current || !!accountStopFlagsRef.current.get(accountId))) break;
const response = await ExtractTimeline({
username: cleanUsername,
auth_token: authToken.trim(),
@@ -1336,6 +1341,7 @@ function App() {
if (accountStopFlagsRef.current.get(accountId)) {
break;
}
if (!await waitForFetch(settings.fetchDelaySeconds, settings.fetchDelayJitterSeconds, () => stopAllRef.current || !!accountStopFlagsRef.current.get(accountId))) break;
const response = await ExtractTimeline({
username: cleanUsername,
auth_token: authToken.trim(),
+24
View File
@@ -743,6 +743,30 @@ export function SettingsPage({ onUnsavedChangesChange, onResetRequest }: Setting
<InputWithContext id="proxy" value={tempSettings.proxy || ""} onChange={(e) => setTempSettings((prev) => ({ ...prev, proxy: e.target.value }))} placeholder="http://proxy1:port, socks5://proxy2:port (optional)" className="max-w-md"/>
</div>
<div className="space-y-2">
<Label>Fetch Delay</Label>
<div className="flex flex-wrap gap-4">
{([
["fetchDelaySeconds", "Base Delay"],
["fetchDelayJitterSeconds", "Random Extra"],
] as const).map(([key, label]) => (
<div key={key} className="space-y-1.5">
<Label htmlFor={key} className="text-xs">{label}</Label>
<div className="flex items-center gap-2">
<InputWithContext id={key} type="number" min="0" max="3600" step="0.1" value={tempSettings[key]} onChange={(e) => {
const value = Number(e.target.value);
setTempSettings((prev) => ({ ...prev, [key]: Number.isFinite(value) ? Math.min(3600, Math.max(0, value)) : 0 }));
}} className="w-24"/>
<span className="text-xs text-muted-foreground">sec</span>
</div>
</div>
))}
</div>
<p className="text-xs text-muted-foreground">Spaces out fetch batch starts, including account switches and retries. Random Extra adds between 0 and the chosen number of seconds. Set both to 0 to disable.</p>
<p className="text-xs text-muted-foreground">Suggested starting point: 5 sec base + 5 sec random extra. No delay guarantees avoidance of rate limits. Waiting counts toward Fetch Timeout; increase it for longer delays.</p>
<p className="text-xs text-muted-foreground">Applies between extractor runs, including date-range fetches, rather than individual requests inside the extractor.</p>
</div>
<div className="space-y-2">
<Label htmlFor="fetch-timeout" className="flex items-center gap-2">
Fetch Timeout
+27
View File
@@ -0,0 +1,27 @@
export function normalizeFetchDelay(value: unknown): number {
return typeof value === "number" && Number.isFinite(value)
? Math.round(Math.min(3600, Math.max(0, value)) * 1000) / 1000
: 0;
}
// One gate shared by all fetch paths, so switching accounts also respects pacing.
export function createFetchDelayGate({
now = () => performance.now(),
random = Math.random,
sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)),
} = {}) {
let lastStarted: number | null = null;
return async (baseSeconds: number, extraSeconds: number, stopped: () => boolean): Promise<boolean> => {
const delay = (normalizeFetchDelay(baseSeconds) + random() * normalizeFetchDelay(extraSeconds)) * 1000;
while (!stopped()) {
const remaining = lastStarted === null ? 0 : lastStarted + delay - now();
if (remaining <= 0) {
lastStarted = now();
return true;
}
// Recheck the shared timestamp after each wait to space concurrent callers too.
await sleep(Math.min(remaining, 100));
}
return false;
};
}
+7
View File
@@ -1,4 +1,5 @@
import { GetDefaults } from "../../wailsjs/go/main/App";
import { normalizeFetchDelay } from "./fetch-delay";
import { normalizeBaseColorName, normalizeThemeName, type BaseColorName, type ThemeName } from "./themes";
export type BuiltInFontFamily = "google-sans" | "inter" | "poppins" | "roboto" | "dm-sans" | "plus-jakarta-sans" | "manrope" | "space-grotesk" | "noto-sans" | "nunito-sans" | "figtree" | "raleway" | "public-sans" | "outfit" | "jetbrains-mono" | "geist-sans" | "bricolage-grotesque";
export type CustomFontFamily = `custom-${string}`;
@@ -171,6 +172,8 @@ export interface Settings {
gifResolution: GifResolution;
proxy: string;
fetchTimeout: number;
fetchDelaySeconds: number;
fetchDelayJitterSeconds: number;
includePhotos: boolean;
includeVideos: boolean;
includeGifs: boolean;
@@ -204,6 +207,8 @@ export const DEFAULT_SETTINGS: Settings = {
gifResolution: "original",
proxy: "",
fetchTimeout: 60,
fetchDelaySeconds: 0,
fetchDelayJitterSeconds: 0,
includePhotos: true,
includeVideos: true,
includeGifs: true,
@@ -550,6 +555,8 @@ function toNormalizedSettings(settings: Partial<Settings>): Settings {
downloadSpeedLimitKBps: normalizeDownloadSpeedLimit(settings.downloadSpeedLimitKBps),
downloadDelaySeconds: normalizeDownloadDelay(settings.downloadDelaySeconds),
downloadDelayJitterSeconds: normalizeDownloadDelay(settings.downloadDelayJitterSeconds),
fetchDelaySeconds: normalizeFetchDelay(settings.fetchDelaySeconds),
fetchDelayJitterSeconds: normalizeFetchDelay(settings.fetchDelayJitterSeconds),
showUpdateNotifications: normalizeBoolean(settings.showUpdateNotifications, DEFAULT_SETTINGS.showUpdateNotifications),
autoConvertGifs: normalizeBoolean(settings.autoConvertGifs, DEFAULT_SETTINGS.autoConvertGifs),
filenameTemplate: normalizeFilenameTemplate(settings.filenameTemplate),