mirror of
https://github.com/Alvin-Zilverstand/Twitter-X-Media-Batch-Downloader.git
synced 2026-11-21 13:53:13 +01:00
f
This commit is contained in:
1 parent
9c269bc8f2
commit
6964753e3e
8 files changed
+164
-2
No files matched your search
@@ -34,6 +34,19 @@
|
||||
|
||||
- Also available on the web at [**TwitterDL.app**](https://twitterdl.app)
|
||||
|
||||
## Fetch pacing
|
||||
|
||||
Settings includes separate **Fetch Delay** controls: a base delay and a random
|
||||
extra from zero to the configured value, in seconds. Both default to zero
|
||||
(disabled). A suggested starting point is 5 seconds base plus up to 5 seconds
|
||||
extra; this is not a guaranteed safe rate for Twitter/X.
|
||||
|
||||
The delay spaces out extractor starts across batches, accounts, resumes, retries,
|
||||
and date-range fetches. The first fetch starts immediately. It does not control
|
||||
requests made internally by the external extractor. Stop interrupts a pending
|
||||
wait, and waiting counts toward Fetch Timeout, so increase the timeout when
|
||||
using longer delays.
|
||||
|
||||
## How to Obtain Auth Token
|
||||
|
||||
> [!Warning]
|
||||
|
||||
+2
-2
@@ -730,7 +730,7 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) {
|
||||
args = append(args, "--guest")
|
||||
}
|
||||
|
||||
args = append(args, "--json", "--metadata")
|
||||
args = append(args, "--json", "--metadata", "--no-discord")
|
||||
|
||||
if req.BatchSize > 0 {
|
||||
args = append(args, "--limit", fmt.Sprintf("%d", req.BatchSize))
|
||||
@@ -840,7 +840,7 @@ func ExtractDateRange(req DateRangeRequest) (*TwitterResponse, error) {
|
||||
args = append(args, "--guest")
|
||||
}
|
||||
|
||||
args = append(args, "--json", "--metadata")
|
||||
args = append(args, "--json", "--metadata", "--no-discord")
|
||||
|
||||
if req.Retweets {
|
||||
args = append(args, "--retweets", "include")
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
"version": "0.0.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "node --experimental-strip-types --test scripts/*.test.mjs",
|
||||
"dev": "vite",
|
||||
"build": "tsc -b && vite build",
|
||||
"lint": "eslint .",
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { createFetchDelayGate, normalizeFetchDelay } from "../src/lib/fetch-delay.ts";
|
||||
|
||||
function clock(random = 0.5) {
|
||||
let time = 0;
|
||||
const waits = [];
|
||||
const gate = createFetchDelayGate({
|
||||
now: () => time,
|
||||
random: () => random,
|
||||
sleep: async (ms) => { waits.push(ms); time += ms; },
|
||||
});
|
||||
return { gate, waits, now: () => time, advance: (ms) => { time += ms; } };
|
||||
}
|
||||
|
||||
test("old or invalid settings disable delay; valid settings are bounded", () => {
|
||||
for (const value of [undefined, null, "5", NaN, Infinity, -1]) {
|
||||
assert.equal(normalizeFetchDelay(value), 0);
|
||||
}
|
||||
assert.equal(normalizeFetchDelay(4000), 3600);
|
||||
assert.equal(normalizeFetchDelay(1.2345), 1.235);
|
||||
});
|
||||
|
||||
test("first fetch starts immediately; later batches/accounts share pacing", async () => {
|
||||
const c = clock();
|
||||
assert.equal(await c.gate(5, 5, () => false), true);
|
||||
assert.equal(c.now(), 0);
|
||||
c.advance(2000); // Time spent fetching the previous batch.
|
||||
assert.equal(await c.gate(5, 5, () => false), true);
|
||||
assert.equal(c.now(), 7500);
|
||||
await c.gate(5, 5, () => false);
|
||||
assert.equal(c.now(), 15000);
|
||||
assert.ok(c.waits.every((ms) => ms <= 100));
|
||||
});
|
||||
|
||||
test("disabled pacing and a long fetch do not introduce unnecessary waits", async () => {
|
||||
const c = clock();
|
||||
await c.gate(0, 0, () => false);
|
||||
await c.gate(0, 0, () => false);
|
||||
c.advance(10000);
|
||||
await c.gate(5, 0, () => false);
|
||||
assert.deepEqual(c.waits, []);
|
||||
});
|
||||
|
||||
test("random-only pacing supports both ends of the random range", async () => {
|
||||
for (const random of [0, 0.999]) {
|
||||
const c = clock(random);
|
||||
await c.gate(0, 5, () => false);
|
||||
await c.gate(0, 5, () => false);
|
||||
assert.equal(c.now(), random * 5000);
|
||||
}
|
||||
});
|
||||
|
||||
test("stop or timeout during a wait prevents the next fetch from starting", async () => {
|
||||
const c = clock();
|
||||
await c.gate(5, 0, () => false);
|
||||
assert.equal(await c.gate(5, 0, () => c.now() >= 200), false);
|
||||
assert.equal(c.now(), 200);
|
||||
assert.equal(await c.gate(0, 0, () => true), false);
|
||||
await c.gate(5, 0, () => false);
|
||||
assert.equal(c.now(), 5000); // Cancellation did not reserve a future slot.
|
||||
});
|
||||
|
||||
test("concurrent callers recheck the shared start time after waking", async () => {
|
||||
let time = 0;
|
||||
const sleepers = [];
|
||||
const gate = createFetchDelayGate({
|
||||
now: () => time,
|
||||
random: () => 0,
|
||||
sleep: () => new Promise((resolve) => sleepers.push(resolve)),
|
||||
});
|
||||
await gate(1, 0, () => false);
|
||||
const starts = [];
|
||||
const a = gate(1, 0, () => false).then(() => starts.push(time));
|
||||
const b = gate(1, 0, () => false).then(() => starts.push(time));
|
||||
time = 1000;
|
||||
sleepers.splice(0).forEach((resolve) => resolve());
|
||||
await a;
|
||||
assert.deepEqual(starts, [1000]);
|
||||
time = 2000;
|
||||
sleepers.splice(0).forEach((resolve) => resolve());
|
||||
await b;
|
||||
assert.deepEqual(starts, [1000, 2000]);
|
||||
});
|
||||
@@ -3,6 +3,7 @@ import { TooltipProvider } from "@/components/ui/tooltip";
|
||||
import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { getSettings, applyThemeMode, applyFont } from "@/lib/settings";
|
||||
import { createFetchDelayGate } from "@/lib/fetch-delay";
|
||||
import { applyTheme } from "@/lib/themes";
|
||||
import { openExternal } from "@/lib/utils";
|
||||
import { isNewerVersion } from "@/lib/version";
|
||||
@@ -29,6 +30,7 @@ const HISTORY_KEY = "twitter_media_fetch_history";
|
||||
const MAX_HISTORY = 10;
|
||||
const CURRENT_VERSION = __APP_VERSION__;
|
||||
const BATCH_SIZE = 100;
|
||||
const waitForFetch = createFetchDelayGate();
|
||||
function formatNumberWithComma(num: number): string {
|
||||
return num.toLocaleString();
|
||||
}
|
||||
@@ -446,6 +448,7 @@ function App() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (!await waitForFetch(settings.fetchDelaySeconds, settings.fetchDelayJitterSeconds, () => stopFetchRef.current)) return;
|
||||
const response = await ExtractDateRange({
|
||||
username: username.trim(),
|
||||
auth_token: authToken.trim(),
|
||||
@@ -522,6 +525,7 @@ function App() {
|
||||
}
|
||||
}
|
||||
const batchNum = page + 1;
|
||||
if (!await waitForFetch(settings.fetchDelaySeconds, settings.fetchDelayJitterSeconds, () => stopFetchRef.current)) break;
|
||||
logger.info(`Fetching batch ${batchNum}${cursor ? " (resuming)" : ""}...`);
|
||||
const response = await ExtractTimeline({
|
||||
username: isBookmarks ? "" : username.trim(),
|
||||
@@ -977,6 +981,7 @@ function App() {
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (!await waitForFetch(settings.fetchDelaySeconds, settings.fetchDelayJitterSeconds, () => stopAllRef.current || !!accountStopFlagsRef.current.get(accountId))) break;
|
||||
const response = await ExtractTimeline({
|
||||
username: cleanUsername,
|
||||
auth_token: authToken.trim(),
|
||||
@@ -1336,6 +1341,7 @@ function App() {
|
||||
if (accountStopFlagsRef.current.get(accountId)) {
|
||||
break;
|
||||
}
|
||||
if (!await waitForFetch(settings.fetchDelaySeconds, settings.fetchDelayJitterSeconds, () => stopAllRef.current || !!accountStopFlagsRef.current.get(accountId))) break;
|
||||
const response = await ExtractTimeline({
|
||||
username: cleanUsername,
|
||||
auth_token: authToken.trim(),
|
||||
|
||||
@@ -743,6 +743,30 @@ export function SettingsPage({ onUnsavedChangesChange, onResetRequest }: Setting
|
||||
<InputWithContext id="proxy" value={tempSettings.proxy || ""} onChange={(e) => setTempSettings((prev) => ({ ...prev, proxy: e.target.value }))} placeholder="http://proxy1:port, socks5://proxy2:port (optional)" className="max-w-md"/>
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
<Label>Fetch Delay</Label>
|
||||
<div className="flex flex-wrap gap-4">
|
||||
{([
|
||||
["fetchDelaySeconds", "Base Delay"],
|
||||
["fetchDelayJitterSeconds", "Random Extra"],
|
||||
] as const).map(([key, label]) => (
|
||||
<div key={key} className="space-y-1.5">
|
||||
<Label htmlFor={key} className="text-xs">{label}</Label>
|
||||
<div className="flex items-center gap-2">
|
||||
<InputWithContext id={key} type="number" min="0" max="3600" step="0.1" value={tempSettings[key]} onChange={(e) => {
|
||||
const value = Number(e.target.value);
|
||||
setTempSettings((prev) => ({ ...prev, [key]: Number.isFinite(value) ? Math.min(3600, Math.max(0, value)) : 0 }));
|
||||
}} className="w-24"/>
|
||||
<span className="text-xs text-muted-foreground">sec</span>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
<p className="text-xs text-muted-foreground">Spaces out fetch batch starts, including account switches and retries. Random Extra adds between 0 and the chosen number of seconds. Set both to 0 to disable.</p>
|
||||
<p className="text-xs text-muted-foreground">Suggested starting point: 5 sec base + 5 sec random extra. No delay guarantees avoidance of rate limits. Waiting counts toward Fetch Timeout; increase it for longer delays.</p>
|
||||
<p className="text-xs text-muted-foreground">Applies between extractor runs, including date-range fetches, rather than individual requests inside the extractor.</p>
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="fetch-timeout" className="flex items-center gap-2">
|
||||
Fetch Timeout
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
export function normalizeFetchDelay(value: unknown): number {
|
||||
return typeof value === "number" && Number.isFinite(value)
|
||||
? Math.round(Math.min(3600, Math.max(0, value)) * 1000) / 1000
|
||||
: 0;
|
||||
}
|
||||
|
||||
// One gate shared by all fetch paths, so switching accounts also respects pacing.
|
||||
export function createFetchDelayGate({
|
||||
now = () => performance.now(),
|
||||
random = Math.random,
|
||||
sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)),
|
||||
} = {}) {
|
||||
let lastStarted: number | null = null;
|
||||
return async (baseSeconds: number, extraSeconds: number, stopped: () => boolean): Promise<boolean> => {
|
||||
const delay = (normalizeFetchDelay(baseSeconds) + random() * normalizeFetchDelay(extraSeconds)) * 1000;
|
||||
while (!stopped()) {
|
||||
const remaining = lastStarted === null ? 0 : lastStarted + delay - now();
|
||||
if (remaining <= 0) {
|
||||
lastStarted = now();
|
||||
return true;
|
||||
}
|
||||
// Recheck the shared timestamp after each wait to space concurrent callers too.
|
||||
await sleep(Math.min(remaining, 100));
|
||||
}
|
||||
return false;
|
||||
};
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { GetDefaults } from "../../wailsjs/go/main/App";
|
||||
import { normalizeFetchDelay } from "./fetch-delay";
|
||||
import { normalizeBaseColorName, normalizeThemeName, type BaseColorName, type ThemeName } from "./themes";
|
||||
export type BuiltInFontFamily = "google-sans" | "inter" | "poppins" | "roboto" | "dm-sans" | "plus-jakarta-sans" | "manrope" | "space-grotesk" | "noto-sans" | "nunito-sans" | "figtree" | "raleway" | "public-sans" | "outfit" | "jetbrains-mono" | "geist-sans" | "bricolage-grotesque";
|
||||
export type CustomFontFamily = `custom-${string}`;
|
||||
@@ -171,6 +172,8 @@ export interface Settings {
|
||||
gifResolution: GifResolution;
|
||||
proxy: string;
|
||||
fetchTimeout: number;
|
||||
fetchDelaySeconds: number;
|
||||
fetchDelayJitterSeconds: number;
|
||||
includePhotos: boolean;
|
||||
includeVideos: boolean;
|
||||
includeGifs: boolean;
|
||||
@@ -204,6 +207,8 @@ export const DEFAULT_SETTINGS: Settings = {
|
||||
gifResolution: "original",
|
||||
proxy: "",
|
||||
fetchTimeout: 60,
|
||||
fetchDelaySeconds: 0,
|
||||
fetchDelayJitterSeconds: 0,
|
||||
includePhotos: true,
|
||||
includeVideos: true,
|
||||
includeGifs: true,
|
||||
@@ -550,6 +555,8 @@ function toNormalizedSettings(settings: Partial<Settings>): Settings {
|
||||
downloadSpeedLimitKBps: normalizeDownloadSpeedLimit(settings.downloadSpeedLimitKBps),
|
||||
downloadDelaySeconds: normalizeDownloadDelay(settings.downloadDelaySeconds),
|
||||
downloadDelayJitterSeconds: normalizeDownloadDelay(settings.downloadDelayJitterSeconds),
|
||||
fetchDelaySeconds: normalizeFetchDelay(settings.fetchDelaySeconds),
|
||||
fetchDelayJitterSeconds: normalizeFetchDelay(settings.fetchDelayJitterSeconds),
|
||||
showUpdateNotifications: normalizeBoolean(settings.showUpdateNotifications, DEFAULT_SETTINGS.showUpdateNotifications),
|
||||
autoConvertGifs: normalizeBoolean(settings.autoConvertGifs, DEFAULT_SETTINGS.autoConvertGifs),
|
||||
filenameTemplate: normalizeFilenameTemplate(settings.filenameTemplate),
|
||||
|
||||
Reference in new issue
Block a user