mirror of
https://github.com/Alvin-Zilverstand/Twitter-X-Media-Batch-Downloader.git
synced 2026-11-21 13:53:13 +01:00
1312 lines
64 KiB
TypeScript
1312 lines
64 KiB
TypeScript
import { useState, useEffect, useRef, useCallback } from "react";
|
|
import { TooltipProvider } from "@/components/ui/tooltip";
|
|
import { getSettings, applyThemeMode, applyFont } from "@/lib/settings";
|
|
import { applyTheme } from "@/lib/themes";
|
|
import { toastWithSound as toast } from "@/lib/toast-with-sound";
|
|
import { logger } from "@/lib/logger";
|
|
import { saveFetchState, getFetchState, clearFetchState, getResumableInfo, stateToResponse, mergeTimelines, saveCursor, getCursor, clearCursor, type FetchState, } from "@/lib/fetch-state";
|
|
import { TitleBar } from "@/components/TitleBar";
|
|
import { Sidebar, type PageType } from "@/components/Sidebar";
|
|
import { Header } from "@/components/Header";
|
|
import { SearchBar, type FetchMode, type PrivateType, type FetchType, type MultipleAccount } from "@/components/SearchBar";
|
|
import { MediaList } from "@/components/MediaList";
|
|
import { DatabaseView } from "@/components/DatabaseView";
|
|
import { SettingsPage } from "@/components/SettingsPage";
|
|
import { DebugLoggerPage } from "@/components/DebugLoggerPage";
|
|
import { DependencySetupDialog } from "@/components/DependencySetupDialog";
|
|
import type { HistoryItem } from "@/components/FetchHistory";
|
|
import type { TwitterResponse } from "@/types/api";
|
|
import { ExtractTimeline, ExtractDateRange, SaveAccountToDBWithStatus, CleanupExtractorProcesses, GetAllAccountsFromDB } from "../wailsjs/go/main/App";
|
|
const HISTORY_KEY = "twitter_media_fetch_history";
|
|
const MAX_HISTORY = 10;
|
|
const CURRENT_VERSION = __APP_VERSION__;
|
|
const BATCH_SIZE = 200;
|
|
function formatNumberWithComma(num: number): string {
|
|
return num.toLocaleString();
|
|
}
|
|
function App() {
|
|
const [currentPage, setCurrentPage] = useState<PageType>("main");
|
|
const [username, setUsername] = useState("");
|
|
const [loading, setLoading] = useState(false);
|
|
const [result, setResult] = useState<TwitterResponse | null>(null);
|
|
const [fetchedMediaType, setFetchedMediaType] = useState<string>("all");
|
|
const [hasUpdate, setHasUpdate] = useState(false);
|
|
const [releaseDate, setReleaseDate] = useState<string | null>(null);
|
|
const [fetchHistory, setFetchHistory] = useState<HistoryItem[]>([]);
|
|
const [resumeInfo, setResumeInfo] = useState<{
|
|
canResume: boolean;
|
|
mediaCount: number;
|
|
} | null>(null);
|
|
const [elapsedTime, setElapsedTime] = useState(0);
|
|
const [remainingTime, setRemainingTime] = useState<number | null>(null);
|
|
const [newMediaCount, setNewMediaCount] = useState<number | null>(null);
|
|
const stopFetchRef = useRef(false);
|
|
const fetchStartTimeRef = useRef<number | null>(null);
|
|
const timeoutIntervalRef = useRef<number | null>(null);
|
|
const [fetchType, setFetchType] = useState<FetchType>("single");
|
|
const [multipleAccounts, setMultipleAccounts] = useState<MultipleAccount[]>([]);
|
|
const [isFetchingAll, setIsFetchingAll] = useState(false);
|
|
const [searchMode, setSearchMode] = useState<FetchMode>("public");
|
|
const [searchPrivateType, setSearchPrivateType] = useState<PrivateType>("bookmarks");
|
|
const stopAllRef = useRef(false);
|
|
const accountTimersRef = useRef<Map<string, number>>(new Map());
|
|
const accountStartTimesRef = useRef<Map<string, number>>(new Map());
|
|
const accountStopFlagsRef = useRef<Map<string, boolean>>(new Map());
|
|
const accountMediaCountRef = useRef<Map<string, number>>(new Map());
|
|
const accountTimeoutSecondsRef = useRef<Map<string, number>>(new Map());
|
|
const dbSaveWarningShownRef = useRef(false);
|
|
const reportDatabaseSaveError = (scope: string, error: unknown) => {
|
|
console.error(`Failed to save ${scope} to database:`, error);
|
|
if (!dbSaveWarningShownRef.current) {
|
|
dbSaveWarningShownRef.current = true;
|
|
toast.warning("Fetch completed, but saving to Saved Accounts failed");
|
|
}
|
|
};
|
|
const handleFetchTypeChange = (type: FetchType) => {
|
|
setFetchType(type);
|
|
if (type === "single") {
|
|
setMultipleAccounts([]);
|
|
setIsFetchingAll(false);
|
|
stopAllRef.current = true;
|
|
accountTimersRef.current.forEach((interval) => clearInterval(interval));
|
|
accountTimersRef.current.clear();
|
|
accountStartTimesRef.current.clear();
|
|
accountTimeoutSecondsRef.current.clear();
|
|
accountStopFlagsRef.current.clear();
|
|
accountMediaCountRef.current.clear();
|
|
}
|
|
else {
|
|
setResult(null);
|
|
}
|
|
};
|
|
useEffect(() => {
|
|
const settings = getSettings();
|
|
applyThemeMode(settings.themeMode);
|
|
applyTheme(settings.theme);
|
|
applyFont(settings.fontFamily);
|
|
const mediaQuery = window.matchMedia("(prefers-color-scheme: dark)");
|
|
const handleChange = () => {
|
|
const currentSettings = getSettings();
|
|
if (currentSettings.themeMode === "auto") {
|
|
applyThemeMode("auto");
|
|
applyTheme(currentSettings.theme);
|
|
}
|
|
};
|
|
mediaQuery.addEventListener("change", handleChange);
|
|
checkForUpdates();
|
|
loadHistory();
|
|
return () => {
|
|
mediaQuery.removeEventListener("change", handleChange);
|
|
};
|
|
}, []);
|
|
const checkForUpdates = async () => {
|
|
try {
|
|
const response = await fetch("https://api.github.com/repos/afkarxyz/Twitter-X-Media-Batch-Downloader/releases/latest");
|
|
const data = await response.json();
|
|
const latestVersion = data.tag_name?.replace(/^v/, "") || "";
|
|
if (data.published_at) {
|
|
setReleaseDate(data.published_at);
|
|
}
|
|
if (latestVersion && latestVersion > CURRENT_VERSION) {
|
|
setHasUpdate(true);
|
|
}
|
|
}
|
|
catch (err) {
|
|
console.error("Failed to check for updates:", err);
|
|
}
|
|
};
|
|
const loadHistory = () => {
|
|
try {
|
|
const saved = localStorage.getItem(HISTORY_KEY);
|
|
if (saved) {
|
|
setFetchHistory(JSON.parse(saved));
|
|
}
|
|
}
|
|
catch (err) {
|
|
console.error("Failed to load history:", err);
|
|
}
|
|
};
|
|
const saveHistory = (history: HistoryItem[]) => {
|
|
try {
|
|
localStorage.setItem(HISTORY_KEY, JSON.stringify(history));
|
|
}
|
|
catch (err) {
|
|
console.error("Failed to save history:", err);
|
|
}
|
|
};
|
|
const addToHistory = (data: TwitterResponse, inputUsername: string) => {
|
|
let cleanUsername = inputUsername.trim();
|
|
if (cleanUsername.startsWith("@")) {
|
|
cleanUsername = cleanUsername.slice(1);
|
|
}
|
|
if (cleanUsername.includes("x.com/") || cleanUsername.includes("twitter.com/")) {
|
|
const match = cleanUsername.match(/(?:x\.com|twitter\.com)\/([^/?]+)/);
|
|
if (match)
|
|
cleanUsername = match[1];
|
|
}
|
|
setFetchHistory((prev) => {
|
|
const apiUsername = data.account_info.name;
|
|
const filtered = prev.filter((h) => h.username.toLowerCase() !== apiUsername.toLowerCase());
|
|
const newItem: HistoryItem = {
|
|
id: crypto.randomUUID(),
|
|
username: apiUsername,
|
|
name: data.account_info.nick,
|
|
image: data.account_info.profile_image,
|
|
mediaCount: data.total_urls,
|
|
timestamp: Date.now(),
|
|
};
|
|
const updated = [newItem, ...filtered].slice(0, MAX_HISTORY);
|
|
saveHistory(updated);
|
|
return updated;
|
|
});
|
|
};
|
|
const removeFromHistory = (id: string) => {
|
|
setFetchHistory((prev) => {
|
|
const updated = prev.filter((h) => h.id !== id);
|
|
saveHistory(updated);
|
|
return updated;
|
|
});
|
|
};
|
|
const handleHistorySelect = (item: HistoryItem) => {
|
|
setUsername(item.username);
|
|
};
|
|
const checkResumable = useCallback((user: string) => {
|
|
if (!user.trim()) {
|
|
setResumeInfo(null);
|
|
return;
|
|
}
|
|
const info = getResumableInfo(user.trim());
|
|
setResumeInfo(info.canResume ? { canResume: true, mediaCount: info.mediaCount } : null);
|
|
}, []);
|
|
useEffect(() => {
|
|
checkResumable(username);
|
|
}, [username, checkResumable]);
|
|
const handleStopFetch = useCallback(async () => {
|
|
stopFetchRef.current = true;
|
|
logger.info("Stopping fetch...");
|
|
toast.info("Stopping...");
|
|
try {
|
|
await CleanupExtractorProcesses();
|
|
}
|
|
catch {
|
|
}
|
|
}, []);
|
|
useEffect(() => {
|
|
if (loading && fetchStartTimeRef.current !== null) {
|
|
const settings = getSettings();
|
|
const timeoutSeconds = settings.fetchTimeout || 60;
|
|
timeoutIntervalRef.current = window.setInterval(() => {
|
|
if (fetchStartTimeRef.current !== null) {
|
|
const elapsed = Math.floor((Date.now() - fetchStartTimeRef.current) / 1000);
|
|
setElapsedTime(elapsed);
|
|
const remaining = Math.max(0, timeoutSeconds - elapsed);
|
|
setRemainingTime(remaining);
|
|
if (remaining <= 0) {
|
|
stopFetchRef.current = true;
|
|
logger.warning("Fetch timeout reached. Stopping...");
|
|
toast.warning("Fetch timeout reached. Stopping...");
|
|
handleStopFetch();
|
|
}
|
|
}
|
|
}, 1000);
|
|
}
|
|
else {
|
|
if (timeoutIntervalRef.current !== null) {
|
|
clearInterval(timeoutIntervalRef.current);
|
|
timeoutIntervalRef.current = null;
|
|
}
|
|
setElapsedTime(0);
|
|
setRemainingTime(null);
|
|
fetchStartTimeRef.current = null;
|
|
}
|
|
return () => {
|
|
if (timeoutIntervalRef.current !== null) {
|
|
clearInterval(timeoutIntervalRef.current);
|
|
timeoutIntervalRef.current = null;
|
|
}
|
|
};
|
|
}, [loading, handleStopFetch]);
|
|
const handleFetch = async (useDateRange: boolean, startDate?: string, endDate?: string, mediaType?: string, retweets?: boolean, mode: FetchMode = "public", privateType?: PrivateType, authToken?: string, isResume?: boolean) => {
|
|
if (loading) {
|
|
logger.warning("Fetch already in progress, please wait or stop first");
|
|
toast.warning("Fetch already in progress");
|
|
return;
|
|
}
|
|
const isBookmarks = mode === "private" && privateType === "bookmarks";
|
|
const isLikes = mode === "private" && privateType === "likes";
|
|
if (!isBookmarks && !username.trim()) {
|
|
toast.error("Please enter a username");
|
|
return;
|
|
}
|
|
if (!authToken?.trim()) {
|
|
toast.error("Please enter your auth token");
|
|
return;
|
|
}
|
|
setLoading(true);
|
|
setFetchedMediaType(mediaType || "all");
|
|
stopFetchRef.current = false;
|
|
fetchStartTimeRef.current = Date.now();
|
|
setElapsedTime(0);
|
|
setNewMediaCount(null);
|
|
const settings = getSettings();
|
|
const timeoutSeconds = settings.fetchTimeout || 60;
|
|
setRemainingTime(timeoutSeconds);
|
|
try {
|
|
await CleanupExtractorProcesses();
|
|
}
|
|
catch {
|
|
}
|
|
const fetchTarget = isBookmarks
|
|
? "your bookmarks"
|
|
: isLikes
|
|
? "your likes"
|
|
: `@${username}`;
|
|
const cleanUsername = isBookmarks ? "bookmarks" : isLikes ? "likes" : username.trim();
|
|
let existingState: FetchState | null = null;
|
|
let cursor: string | undefined;
|
|
let allTimeline: TwitterResponse["timeline"] = [];
|
|
let accountInfo: TwitterResponse["account_info"] | null = null;
|
|
if (isResume) {
|
|
existingState = getFetchState(cleanUsername);
|
|
if (existingState && existingState.cursor && !existingState.completed) {
|
|
cursor = existingState.cursor;
|
|
allTimeline = existingState.timeline;
|
|
accountInfo = existingState.accountInfo;
|
|
logger.info(`Resuming ${fetchTarget} from ${allTimeline.length} items...`);
|
|
if (accountInfo) {
|
|
setResult({
|
|
account_info: accountInfo,
|
|
timeline: allTimeline,
|
|
total_urls: allTimeline.length,
|
|
metadata: { new_entries: 0, page: 0, batch_size: 0, has_more: true },
|
|
});
|
|
}
|
|
}
|
|
else {
|
|
toast.error("No resumable fetch found");
|
|
setLoading(false);
|
|
return;
|
|
}
|
|
}
|
|
else {
|
|
clearFetchState(cleanUsername);
|
|
setResult(null);
|
|
logger.info(`Fetching ${fetchTarget}...`);
|
|
}
|
|
try {
|
|
let finalData: TwitterResponse | null = null;
|
|
if (useDateRange && startDate && endDate && mode === "public") {
|
|
logger.info(`Using date range: ${startDate} to ${endDate}`);
|
|
if (fetchStartTimeRef.current !== null) {
|
|
const settings = getSettings();
|
|
const timeoutSeconds = settings.fetchTimeout || 60;
|
|
const elapsed = Math.floor((Date.now() - fetchStartTimeRef.current) / 1000);
|
|
if (elapsed >= timeoutSeconds) {
|
|
logger.warning(`Timeout reached (${timeoutSeconds}s). Stopping fetch...`);
|
|
stopFetchRef.current = true;
|
|
toast.warning("Fetch timeout reached. Stopping...");
|
|
setLoading(false);
|
|
return;
|
|
}
|
|
}
|
|
const response = await ExtractDateRange({
|
|
username: username.trim(),
|
|
auth_token: authToken.trim(),
|
|
start_date: startDate,
|
|
end_date: endDate,
|
|
media_filter: mediaType || "all",
|
|
retweets: retweets || false,
|
|
});
|
|
finalData = JSON.parse(response);
|
|
if (finalData && finalData.account_info) {
|
|
try {
|
|
await SaveAccountToDBWithStatus(finalData.account_info.name, finalData.account_info.nick, finalData.account_info.profile_image, finalData.total_urls, JSON.stringify(finalData), mediaType || "all", finalData.cursor || "", finalData.completed ?? true);
|
|
}
|
|
catch (err) {
|
|
reportDatabaseSaveError("date range data", err);
|
|
}
|
|
}
|
|
if (finalData && finalData.account_info) {
|
|
try {
|
|
await SaveAccountToDBWithStatus(finalData.account_info.name, finalData.account_info.nick, finalData.account_info.profile_image, finalData.total_urls, JSON.stringify(finalData), mediaType || "all", finalData.cursor || "", finalData.completed ?? true);
|
|
}
|
|
catch (err) {
|
|
reportDatabaseSaveError("date range data", err);
|
|
}
|
|
}
|
|
}
|
|
else {
|
|
const settings = getSettings();
|
|
const isSingleMode = settings.fetchMode === "single";
|
|
const batchSize = isSingleMode ? 0 : BATCH_SIZE;
|
|
let timelineType = "timeline";
|
|
if (isBookmarks) {
|
|
timelineType = "bookmarks";
|
|
}
|
|
else if (isLikes) {
|
|
timelineType = "likes";
|
|
}
|
|
let hasMore = true;
|
|
let page = 0;
|
|
while (hasMore && !stopFetchRef.current) {
|
|
if (fetchStartTimeRef.current !== null) {
|
|
const settings = getSettings();
|
|
const timeoutSeconds = settings.fetchTimeout || 60;
|
|
const elapsed = Math.floor((Date.now() - fetchStartTimeRef.current) / 1000);
|
|
if (elapsed >= timeoutSeconds) {
|
|
logger.warning(`Timeout reached (${timeoutSeconds}s). Stopping fetch...`);
|
|
stopFetchRef.current = true;
|
|
toast.warning("Fetch timeout reached. Stopping...");
|
|
if (accountInfo && allTimeline.length > 0) {
|
|
saveFetchState({
|
|
username: cleanUsername,
|
|
cursor: cursor || "",
|
|
timeline: allTimeline,
|
|
accountInfo: accountInfo,
|
|
totalFetched: allTimeline.length,
|
|
completed: false,
|
|
authToken: authToken.trim(),
|
|
mediaType: mediaType || "all",
|
|
retweets: retweets || false,
|
|
timelineType: timelineType,
|
|
});
|
|
const timeoutResponse: TwitterResponse = {
|
|
account_info: accountInfo,
|
|
timeline: allTimeline,
|
|
total_urls: allTimeline.length,
|
|
metadata: {
|
|
new_entries: 0,
|
|
page: page,
|
|
batch_size: batchSize,
|
|
has_more: hasMore,
|
|
cursor: cursor,
|
|
completed: false,
|
|
},
|
|
cursor: cursor,
|
|
completed: false,
|
|
};
|
|
try {
|
|
await SaveAccountToDBWithStatus(accountInfo.name, accountInfo.nick, accountInfo.profile_image, allTimeline.length, JSON.stringify(timeoutResponse), mediaType || "all", cursor || "", false);
|
|
logger.info(`Saved ${allTimeline.length} items before timeout`);
|
|
}
|
|
catch (err) {
|
|
reportDatabaseSaveError("timeout state", err);
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
const batchNum = page + 1;
|
|
logger.info(isSingleMode ? "Fetching all media..." : `Fetching batch ${batchNum}${cursor ? " (resuming)" : ""}...`);
|
|
const response = await ExtractTimeline({
|
|
username: isBookmarks ? "" : username.trim(),
|
|
auth_token: authToken.trim(),
|
|
timeline_type: timelineType,
|
|
batch_size: batchSize,
|
|
page: page,
|
|
media_type: mediaType || "all",
|
|
retweets: retweets || false,
|
|
cursor: cursor,
|
|
});
|
|
const data: TwitterResponse = JSON.parse(response);
|
|
if (!accountInfo && data.account_info) {
|
|
accountInfo = data.account_info;
|
|
if (isBookmarks) {
|
|
accountInfo.name = "bookmarks";
|
|
accountInfo.nick = "My Bookmarks";
|
|
}
|
|
else if (isLikes) {
|
|
accountInfo.name = "likes";
|
|
accountInfo.nick = "My Likes";
|
|
}
|
|
}
|
|
const previousCount = allTimeline.length;
|
|
allTimeline = mergeTimelines(allTimeline, data.timeline);
|
|
const newCount = allTimeline.length - previousCount;
|
|
if (newCount > 0) {
|
|
setNewMediaCount(newCount);
|
|
setTimeout(() => setNewMediaCount(null), 1000);
|
|
}
|
|
cursor = data.cursor;
|
|
hasMore = !!data.cursor && !data.completed;
|
|
page++;
|
|
if (cursor) {
|
|
saveCursor(cleanUsername, cursor);
|
|
}
|
|
if (accountInfo) {
|
|
setResult({
|
|
account_info: accountInfo,
|
|
timeline: allTimeline,
|
|
total_urls: allTimeline.length,
|
|
metadata: {
|
|
new_entries: data.timeline.length,
|
|
page: page,
|
|
batch_size: batchSize,
|
|
has_more: hasMore,
|
|
cursor: cursor,
|
|
completed: !hasMore,
|
|
},
|
|
cursor: cursor,
|
|
completed: !hasMore,
|
|
});
|
|
}
|
|
logger.info(`Fetched ${allTimeline.length} items total`);
|
|
saveFetchState({
|
|
username: cleanUsername,
|
|
cursor: cursor || "",
|
|
timeline: allTimeline,
|
|
accountInfo: accountInfo,
|
|
totalFetched: allTimeline.length,
|
|
completed: !hasMore,
|
|
authToken: authToken.trim(),
|
|
mediaType: mediaType || "all",
|
|
retweets: retweets || false,
|
|
timelineType: timelineType,
|
|
});
|
|
if (accountInfo) {
|
|
const currentResponse: TwitterResponse = {
|
|
account_info: accountInfo,
|
|
timeline: allTimeline,
|
|
total_urls: allTimeline.length,
|
|
metadata: {
|
|
new_entries: data.timeline.length,
|
|
page: page,
|
|
batch_size: batchSize,
|
|
has_more: hasMore,
|
|
cursor: cursor,
|
|
completed: !hasMore,
|
|
},
|
|
cursor: cursor,
|
|
completed: !hasMore,
|
|
};
|
|
try {
|
|
await SaveAccountToDBWithStatus(accountInfo.name, accountInfo.nick, accountInfo.profile_image, allTimeline.length, JSON.stringify(currentResponse), mediaType || "all", cursor || "", !hasMore);
|
|
}
|
|
catch (err) {
|
|
reportDatabaseSaveError("fetch progress", err);
|
|
}
|
|
}
|
|
}
|
|
if (stopFetchRef.current) {
|
|
const elapsedSecs = fetchStartTimeRef.current ? Math.floor((Date.now() - fetchStartTimeRef.current) / 1000) : 0;
|
|
logger.info(`Stopped at ${allTimeline.length} items - can resume later (${elapsedSecs}s)`);
|
|
toast.info(`Stopped at ${formatNumberWithComma(allTimeline.length)} items`);
|
|
setResumeInfo({ canResume: true, mediaCount: allTimeline.length });
|
|
}
|
|
else {
|
|
clearFetchState(cleanUsername);
|
|
clearCursor(cleanUsername);
|
|
setResumeInfo(null);
|
|
}
|
|
finalData = accountInfo
|
|
? {
|
|
account_info: accountInfo,
|
|
timeline: allTimeline,
|
|
total_urls: allTimeline.length,
|
|
metadata: {
|
|
new_entries: allTimeline.length,
|
|
page: page,
|
|
batch_size: batchSize,
|
|
has_more: false,
|
|
cursor: cursor,
|
|
completed: !stopFetchRef.current,
|
|
},
|
|
cursor: cursor,
|
|
completed: !stopFetchRef.current,
|
|
}
|
|
: null;
|
|
}
|
|
if (finalData) {
|
|
setResult(finalData);
|
|
if (mode === "public") {
|
|
addToHistory(finalData, username);
|
|
}
|
|
if (!useDateRange) {
|
|
try {
|
|
await SaveAccountToDBWithStatus(finalData.account_info.name, finalData.account_info.nick, finalData.account_info.profile_image, finalData.total_urls, JSON.stringify(finalData), mediaType || "all", finalData.cursor || "", finalData.completed ?? true);
|
|
}
|
|
catch (err) {
|
|
reportDatabaseSaveError("final fetch status", err);
|
|
}
|
|
}
|
|
if (!stopFetchRef.current) {
|
|
const elapsedSecs = fetchStartTimeRef.current ? Math.floor((Date.now() - fetchStartTimeRef.current) / 1000) : 0;
|
|
logger.success(`Found ${finalData.total_urls} media items (${elapsedSecs}s)`);
|
|
toast.success(`${finalData.total_urls} media items found`);
|
|
}
|
|
}
|
|
}
|
|
catch (error) {
|
|
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
const elapsedSecs = fetchStartTimeRef.current ? Math.floor((Date.now() - fetchStartTimeRef.current) / 1000) : 0;
|
|
logger.error(`Failed to fetch: ${errorMsg} (${elapsedSecs}s)`);
|
|
if (errorMsg.toLowerCase().includes("xtractor dependency") || errorMsg.toLowerCase().includes("extractor dependency")) {
|
|
toast.error(errorMsg);
|
|
}
|
|
else {
|
|
toast.error("Failed to fetch media");
|
|
}
|
|
const savedState = getFetchState(cleanUsername);
|
|
if (savedState && savedState.timeline.length > 0) {
|
|
const partialResponse = stateToResponse(savedState);
|
|
if (partialResponse) {
|
|
setResult(partialResponse);
|
|
setResumeInfo({ canResume: true, mediaCount: savedState.timeline.length });
|
|
toast.info(`Saved ${formatNumberWithComma(savedState.timeline.length)} items - can resume`);
|
|
try {
|
|
await SaveAccountToDBWithStatus(partialResponse.account_info.name, partialResponse.account_info.nick, partialResponse.account_info.profile_image, partialResponse.total_urls, JSON.stringify(partialResponse), mediaType || "all", savedState.cursor || "", false);
|
|
}
|
|
catch (dbErr) {
|
|
reportDatabaseSaveError("partial fetch data", dbErr);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
finally {
|
|
setLoading(false);
|
|
fetchStartTimeRef.current = null;
|
|
setElapsedTime(0);
|
|
setRemainingTime(null);
|
|
}
|
|
};
|
|
const handleResume = (authToken: string, mediaType?: string, retweets?: boolean) => {
|
|
handleFetch(false, undefined, undefined, mediaType, retweets, "public", undefined, authToken, true);
|
|
};
|
|
const handleClearResume = () => {
|
|
if (username.trim()) {
|
|
clearFetchState(username.trim());
|
|
setResumeInfo(null);
|
|
toast.info("Resume data cleared");
|
|
}
|
|
};
|
|
const handleLoadFromDB = (responseJSON: string, loadedUsername: string) => {
|
|
try {
|
|
const data: TwitterResponse = JSON.parse(responseJSON);
|
|
setResult(data);
|
|
setUsername(loadedUsername);
|
|
setCurrentPage("main");
|
|
setFetchType("single");
|
|
const isPrivate = loadedUsername === "bookmarks" || loadedUsername === "likes";
|
|
if (isPrivate) {
|
|
if (loadedUsername === "bookmarks") {
|
|
setSearchMode("private");
|
|
setSearchPrivateType("bookmarks");
|
|
}
|
|
else if (loadedUsername === "likes") {
|
|
setSearchMode("private");
|
|
setSearchPrivateType("likes");
|
|
}
|
|
}
|
|
else {
|
|
setSearchMode("public");
|
|
}
|
|
toast.success(`Loaded @${loadedUsername} from database`);
|
|
}
|
|
catch (error) {
|
|
toast.error("Failed to parse saved data");
|
|
}
|
|
};
|
|
const parseUsername = (input: string): string => {
|
|
let clean = input.trim();
|
|
if (clean.startsWith("@")) {
|
|
clean = clean.slice(1);
|
|
}
|
|
if (clean.includes("x.com/") || clean.includes("twitter.com/")) {
|
|
const match = clean.match(/(?:x\.com|twitter\.com)\/([^/?]+)/);
|
|
if (match)
|
|
clean = match[1];
|
|
}
|
|
return clean;
|
|
};
|
|
const handleUpdateSelected = (usernames: string[]) => {
|
|
if (usernames.length === 0) {
|
|
toast.error("No accounts selected");
|
|
return;
|
|
}
|
|
const accounts: MultipleAccount[] = usernames.map((username) => {
|
|
const cleanUsername = parseUsername(username);
|
|
return {
|
|
id: crypto.randomUUID(),
|
|
username: cleanUsername,
|
|
status: "pending",
|
|
mediaCount: 0,
|
|
previousMediaCount: 0,
|
|
elapsedTime: 0,
|
|
remainingTime: null,
|
|
showDiff: false,
|
|
};
|
|
});
|
|
setMultipleAccounts((prev) => {
|
|
const existingUsernames = new Set(prev.map((acc) => acc.username.toLowerCase()));
|
|
const newAccounts = accounts.filter((acc) => !existingUsernames.has(acc.username.toLowerCase()));
|
|
return [...prev, ...newAccounts];
|
|
});
|
|
setFetchType("multiple");
|
|
setCurrentPage("main");
|
|
toast.success(`Added ${formatNumberWithComma(accounts.length)} account(s) to multiple fetch`);
|
|
};
|
|
const handleImportFile = () => {
|
|
const input = document.createElement("input");
|
|
input.type = "file";
|
|
input.accept = ".txt";
|
|
input.onchange = async (e) => {
|
|
const file = (e.target as HTMLInputElement).files?.[0];
|
|
if (!file)
|
|
return;
|
|
try {
|
|
const text = await file.text();
|
|
const lines = text.split("\n").map((line) => line.trim()).filter((line) => line.length > 0);
|
|
if (lines.length === 0) {
|
|
toast.error("File is empty");
|
|
return;
|
|
}
|
|
const accounts: MultipleAccount[] = lines.map((line) => {
|
|
const username = parseUsername(line);
|
|
return {
|
|
id: crypto.randomUUID(),
|
|
username,
|
|
status: "pending",
|
|
mediaCount: 0,
|
|
previousMediaCount: 0,
|
|
elapsedTime: 0,
|
|
remainingTime: null,
|
|
showDiff: false,
|
|
};
|
|
});
|
|
setMultipleAccounts(accounts);
|
|
toast.success(`Imported ${formatNumberWithComma(accounts.length)} account(s)`);
|
|
}
|
|
catch (error) {
|
|
toast.error("Failed to read file");
|
|
}
|
|
};
|
|
input.click();
|
|
};
|
|
const handleFetchAll = async () => {
|
|
if (multipleAccounts.length === 0) {
|
|
toast.error("No accounts to fetch");
|
|
return;
|
|
}
|
|
setIsFetchingAll(true);
|
|
stopAllRef.current = false;
|
|
const PUBLIC_AUTH_TOKEN_KEY = "twitter_public_auth_token";
|
|
const authToken = localStorage.getItem(PUBLIC_AUTH_TOKEN_KEY) || "";
|
|
if (!authToken.trim()) {
|
|
toast.error("Please enter your auth token");
|
|
setIsFetchingAll(false);
|
|
return;
|
|
}
|
|
const settings = getSettings();
|
|
const timeoutSeconds = settings.fetchTimeout || 60;
|
|
setMultipleAccounts((prev) => prev.map((acc) => ({
|
|
...acc,
|
|
status: "pending" as const,
|
|
mediaCount: 0,
|
|
previousMediaCount: 0,
|
|
elapsedTime: 0,
|
|
remainingTime: timeoutSeconds,
|
|
error: undefined,
|
|
showDiff: false,
|
|
})));
|
|
let currentAccounts = [...multipleAccounts];
|
|
for (let i = 0; i < currentAccounts.length; i++) {
|
|
if (stopAllRef.current) {
|
|
break;
|
|
}
|
|
const account = currentAccounts[i];
|
|
const accountId = account.id;
|
|
accountStopFlagsRef.current.set(accountId, false);
|
|
setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId
|
|
? { ...acc, status: "fetching" as const }
|
|
: acc));
|
|
const settings = getSettings();
|
|
const timeoutSeconds = settings.fetchTimeout || 60;
|
|
accountStartTimesRef.current.set(accountId, Date.now());
|
|
accountTimeoutSecondsRef.current.set(accountId, timeoutSeconds);
|
|
setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId
|
|
? { ...acc, elapsedTime: 0, remainingTime: timeoutSeconds }
|
|
: acc));
|
|
const timerInterval = window.setInterval(() => {
|
|
const startTime = accountStartTimesRef.current.get(accountId);
|
|
const timeoutSecs = accountTimeoutSecondsRef.current.get(accountId) || timeoutSeconds;
|
|
if (startTime) {
|
|
const elapsed = Math.floor((Date.now() - startTime) / 1000);
|
|
const remaining = Math.max(0, timeoutSecs - elapsed);
|
|
setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId
|
|
? { ...acc, elapsedTime: elapsed, remainingTime: remaining }
|
|
: acc));
|
|
if (remaining <= 0) {
|
|
accountStopFlagsRef.current.set(accountId, true);
|
|
clearInterval(timerInterval);
|
|
accountTimersRef.current.delete(accountId);
|
|
accountStartTimesRef.current.delete(accountId);
|
|
accountTimeoutSecondsRef.current.delete(accountId);
|
|
const mediaCount = accountMediaCountRef.current.get(accountId) || 0;
|
|
setMultipleAccounts((prev) => prev.map((acc) => {
|
|
if (acc.id === accountId) {
|
|
const status = mediaCount === 0 ? ("failed" as const) : ("incomplete" as const);
|
|
return { ...acc, status, remainingTime: 0, mediaCount };
|
|
}
|
|
return acc;
|
|
}));
|
|
CleanupExtractorProcesses().catch(() => { });
|
|
}
|
|
}
|
|
}, 1000);
|
|
accountTimersRef.current.set(accountId, timerInterval);
|
|
try {
|
|
if (stopAllRef.current) {
|
|
clearInterval(timerInterval);
|
|
accountTimersRef.current.delete(accountId);
|
|
accountStartTimesRef.current.delete(accountId);
|
|
accountTimeoutSecondsRef.current.delete(accountId);
|
|
setMultipleAccounts((prev) => prev.map((acc) => {
|
|
if (acc.id === accountId) {
|
|
const status = acc.mediaCount === 0 ? ("failed" as const) : ("incomplete" as const);
|
|
return { ...acc, status };
|
|
}
|
|
return acc;
|
|
}));
|
|
continue;
|
|
}
|
|
const fetchSettings = getSettings();
|
|
const isSingleModeMultiple = fetchSettings.fetchMode === "single";
|
|
const batchSizeMultiple = isSingleModeMultiple ? 0 : BATCH_SIZE;
|
|
const cleanUsername = account.username.trim();
|
|
let allTimeline: TwitterResponse["timeline"] = [];
|
|
let accountInfo: TwitterResponse["account_info"] | null = null;
|
|
let cursor: string | undefined;
|
|
let hasMore = true;
|
|
let page = 0;
|
|
let previousMediaCount = account.mediaCount || 0;
|
|
while (hasMore && !stopAllRef.current) {
|
|
if (accountStopFlagsRef.current.get(accountId)) {
|
|
if (accountInfo && allTimeline.length > 0) {
|
|
saveFetchState({
|
|
username: cleanUsername,
|
|
cursor: cursor || "",
|
|
timeline: allTimeline,
|
|
accountInfo: accountInfo,
|
|
totalFetched: allTimeline.length,
|
|
completed: false,
|
|
authToken: authToken.trim(),
|
|
mediaType: fetchedMediaType || "all",
|
|
retweets: false,
|
|
timelineType: "timeline",
|
|
});
|
|
try {
|
|
await SaveAccountToDBWithStatus(accountInfo.name, accountInfo.nick, accountInfo.profile_image, allTimeline.length, JSON.stringify({
|
|
account_info: accountInfo,
|
|
timeline: allTimeline,
|
|
total_urls: allTimeline.length,
|
|
metadata: {
|
|
new_entries: 0,
|
|
page: page,
|
|
batch_size: batchSizeMultiple,
|
|
has_more: hasMore,
|
|
cursor: cursor,
|
|
completed: false,
|
|
},
|
|
cursor: cursor,
|
|
completed: false,
|
|
}), fetchedMediaType || "all", cursor || "", false);
|
|
}
|
|
catch (err) {
|
|
reportDatabaseSaveError("multi-account timeout state", err);
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
const response = await ExtractTimeline({
|
|
username: cleanUsername,
|
|
auth_token: authToken.trim(),
|
|
timeline_type: "timeline",
|
|
batch_size: batchSizeMultiple,
|
|
page: page,
|
|
media_type: fetchedMediaType || "all",
|
|
retweets: false,
|
|
cursor: cursor,
|
|
});
|
|
const data: TwitterResponse = JSON.parse(response);
|
|
if (!accountInfo && data.account_info) {
|
|
accountInfo = data.account_info;
|
|
}
|
|
allTimeline = mergeTimelines(allTimeline, data.timeline);
|
|
cursor = data.cursor;
|
|
hasMore = !!data.cursor && !data.completed;
|
|
page++;
|
|
if (cursor) {
|
|
saveCursor(cleanUsername, cursor);
|
|
}
|
|
const currentMediaCount = allTimeline.length;
|
|
accountMediaCountRef.current.set(accountId, currentMediaCount);
|
|
const hasNewItems = currentMediaCount > previousMediaCount;
|
|
setMultipleAccounts((prev) => prev.map((acc) => {
|
|
if (acc.id === accountId) {
|
|
return {
|
|
...acc,
|
|
accountInfo: accountInfo || undefined,
|
|
previousMediaCount: previousMediaCount,
|
|
mediaCount: currentMediaCount,
|
|
showDiff: hasNewItems,
|
|
cursor: cursor,
|
|
};
|
|
}
|
|
return acc;
|
|
}));
|
|
if (hasNewItems) {
|
|
setTimeout(() => {
|
|
setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId ? { ...acc, showDiff: false } : acc));
|
|
}, 1000);
|
|
}
|
|
previousMediaCount = currentMediaCount;
|
|
if (accountInfo) {
|
|
saveFetchState({
|
|
username: cleanUsername,
|
|
cursor: cursor || "",
|
|
timeline: allTimeline,
|
|
accountInfo: accountInfo,
|
|
totalFetched: allTimeline.length,
|
|
completed: !hasMore,
|
|
authToken: authToken.trim(),
|
|
mediaType: fetchedMediaType || "all",
|
|
retweets: false,
|
|
timelineType: "timeline",
|
|
});
|
|
}
|
|
if (accountInfo) {
|
|
try {
|
|
await SaveAccountToDBWithStatus(accountInfo.name, accountInfo.nick, accountInfo.profile_image, allTimeline.length, JSON.stringify({
|
|
account_info: accountInfo,
|
|
timeline: allTimeline,
|
|
total_urls: allTimeline.length,
|
|
metadata: {
|
|
new_entries: data.timeline.length,
|
|
page: page,
|
|
batch_size: batchSizeMultiple,
|
|
has_more: hasMore,
|
|
cursor: cursor,
|
|
completed: !hasMore,
|
|
},
|
|
cursor: cursor,
|
|
completed: !hasMore,
|
|
}), fetchedMediaType || "all", cursor || "", !hasMore);
|
|
}
|
|
catch (err) {
|
|
reportDatabaseSaveError("multi-account progress", err);
|
|
}
|
|
}
|
|
}
|
|
const startTime = accountStartTimesRef.current.get(accountId);
|
|
const elapsedSecs = startTime ? Math.floor((Date.now() - startTime) / 1000) : 0;
|
|
clearInterval(timerInterval);
|
|
accountTimersRef.current.delete(accountId);
|
|
accountStartTimesRef.current.delete(accountId);
|
|
accountTimeoutSecondsRef.current.delete(accountId);
|
|
const finalMediaCount = allTimeline.length;
|
|
accountMediaCountRef.current.set(accountId, finalMediaCount);
|
|
const wasTimeout = accountStopFlagsRef.current.get(accountId);
|
|
if (wasTimeout) {
|
|
logger.warning(`@${account.username}: timeout - ${finalMediaCount} items (${elapsedSecs}s)`);
|
|
setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId
|
|
? { ...acc, status: finalMediaCount === 0 ? ("failed" as const) : ("incomplete" as const) }
|
|
: acc));
|
|
}
|
|
else if (stopAllRef.current) {
|
|
logger.info(`@${account.username}: stopped - ${finalMediaCount} items (${elapsedSecs}s)`);
|
|
setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId
|
|
? { ...acc, status: finalMediaCount === 0 ? ("failed" as const) : ("incomplete" as const) }
|
|
: acc));
|
|
}
|
|
else if (hasMore) {
|
|
logger.warning(`@${account.username}: incomplete - ${finalMediaCount} items (${elapsedSecs}s)`);
|
|
setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId
|
|
? { ...acc, status: finalMediaCount === 0 ? ("failed" as const) : ("incomplete" as const) }
|
|
: acc));
|
|
}
|
|
else {
|
|
logger.success(`@${account.username}: completed - ${finalMediaCount} items (${elapsedSecs}s)`);
|
|
setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId
|
|
? { ...acc, status: "completed" as const }
|
|
: acc));
|
|
}
|
|
}
|
|
catch (error) {
|
|
const startTime = accountStartTimesRef.current.get(accountId);
|
|
const elapsedSecs = startTime ? Math.floor((Date.now() - startTime) / 1000) : 0;
|
|
const timerInterval = accountTimersRef.current.get(accountId);
|
|
if (timerInterval) {
|
|
clearInterval(timerInterval);
|
|
accountTimersRef.current.delete(accountId);
|
|
}
|
|
accountStartTimesRef.current.delete(accountId);
|
|
accountTimeoutSecondsRef.current.delete(accountId);
|
|
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
const mediaCount = accountMediaCountRef.current.get(accountId) || 0;
|
|
const isAuthError = errorMsg.toLowerCase().includes("401") ||
|
|
errorMsg.toLowerCase().includes("unauthorized") ||
|
|
errorMsg.toLowerCase().includes("auth token may be invalid") ||
|
|
errorMsg.toLowerCase().includes("auth token may be expired") ||
|
|
errorMsg.toLowerCase().includes("invalid or expired");
|
|
const status = isAuthError || mediaCount === 0 ? ("failed" as const) : ("incomplete" as const);
|
|
accountMediaCountRef.current.delete(accountId);
|
|
setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId
|
|
? {
|
|
...acc,
|
|
status,
|
|
error: errorMsg,
|
|
mediaCount,
|
|
}
|
|
: acc));
|
|
logger.error(`@${account.username}: failed - ${errorMsg} (${elapsedSecs}s)`);
|
|
}
|
|
}
|
|
setIsFetchingAll(false);
|
|
if (!stopAllRef.current) {
|
|
toast.success("All accounts fetched");
|
|
}
|
|
};
|
|
const handleStopAll = () => {
|
|
stopAllRef.current = true;
|
|
setIsFetchingAll(false);
|
|
accountTimersRef.current.forEach((interval) => clearInterval(interval));
|
|
accountTimersRef.current.clear();
|
|
accountStartTimesRef.current.clear();
|
|
accountTimeoutSecondsRef.current.clear();
|
|
setMultipleAccounts((prev) => prev.map((acc) => {
|
|
if (acc.status === "fetching") {
|
|
const mediaCount = accountMediaCountRef.current.get(acc.id) || acc.mediaCount || 0;
|
|
const status = mediaCount === 0 ? ("failed" as const) : ("incomplete" as const);
|
|
return { ...acc, status, mediaCount };
|
|
}
|
|
return acc;
|
|
}));
|
|
CleanupExtractorProcesses().catch(() => { });
|
|
toast.info("Stopped all fetches");
|
|
};
|
|
const handleStopAccount = (accountId: string) => {
|
|
accountStopFlagsRef.current.set(accountId, true);
|
|
setMultipleAccounts((prev) => prev.map((acc) => {
|
|
if (acc.id === accountId && acc.status === "fetching") {
|
|
const mediaCount = accountMediaCountRef.current.get(accountId) || acc.mediaCount || 0;
|
|
const status = mediaCount === 0 ? ("failed" as const) : ("incomplete" as const);
|
|
return { ...acc, status, mediaCount };
|
|
}
|
|
return acc;
|
|
}));
|
|
const timerInterval = accountTimersRef.current.get(accountId);
|
|
if (timerInterval) {
|
|
clearInterval(timerInterval);
|
|
accountTimersRef.current.delete(accountId);
|
|
}
|
|
accountStartTimesRef.current.delete(accountId);
|
|
accountTimeoutSecondsRef.current.delete(accountId);
|
|
CleanupExtractorProcesses().catch(() => { });
|
|
toast.info("Stopped account fetch");
|
|
};
|
|
const handleRetryAccount = async (accountId: string) => {
|
|
const account = multipleAccounts.find((acc) => acc.id === accountId);
|
|
if (!account)
|
|
return;
|
|
const PUBLIC_AUTH_TOKEN_KEY = "twitter_public_auth_token";
|
|
const authToken = localStorage.getItem(PUBLIC_AUTH_TOKEN_KEY) || "";
|
|
if (!authToken.trim()) {
|
|
toast.error("Please enter your auth token");
|
|
return;
|
|
}
|
|
const retrySettings = getSettings();
|
|
const isSingleModeRetry = retrySettings.fetchMode === "single";
|
|
const batchSizeRetry = isSingleModeRetry ? 0 : BATCH_SIZE;
|
|
const cleanUsername = account.username.trim();
|
|
let existingState: FetchState | null = null;
|
|
let cursor: string | undefined = account.cursor;
|
|
let allTimeline: TwitterResponse["timeline"] = [];
|
|
let accountInfo: TwitterResponse["account_info"] | null = null;
|
|
let previousMediaCount = account.mediaCount || 0;
|
|
existingState = getFetchState(cleanUsername);
|
|
if (existingState && existingState.cursor && !existingState.completed) {
|
|
cursor = existingState.cursor;
|
|
allTimeline = existingState.timeline;
|
|
if (existingState.accountInfo) {
|
|
accountInfo = existingState.accountInfo;
|
|
}
|
|
previousMediaCount = existingState.timeline.length;
|
|
logger.info(`Resuming @${cleanUsername} from ${allTimeline.length} items...`);
|
|
}
|
|
else {
|
|
if (!cursor) {
|
|
const savedCursor = getCursor(cleanUsername);
|
|
if (savedCursor) {
|
|
cursor = savedCursor;
|
|
previousMediaCount = account.mediaCount || 0;
|
|
logger.info(`Retrying @${cleanUsername} from cursor (${previousMediaCount} items)...`);
|
|
}
|
|
else {
|
|
try {
|
|
const accounts = await GetAllAccountsFromDB();
|
|
const dbAccount = accounts.find((acc) => acc.username.toLowerCase() === cleanUsername.toLowerCase());
|
|
if (dbAccount?.cursor) {
|
|
cursor = dbAccount.cursor;
|
|
previousMediaCount = account.mediaCount || 0;
|
|
logger.info(`Retrying @${cleanUsername} from database cursor (${previousMediaCount} items)...`);
|
|
}
|
|
else {
|
|
clearFetchState(cleanUsername);
|
|
clearCursor(cleanUsername);
|
|
logger.info(`Retrying @${cleanUsername} from beginning...`);
|
|
}
|
|
}
|
|
catch (dbError) {
|
|
console.error("Failed to get cursor from database:", dbError);
|
|
clearFetchState(cleanUsername);
|
|
clearCursor(cleanUsername);
|
|
logger.info(`Retrying @${cleanUsername} from beginning...`);
|
|
}
|
|
}
|
|
}
|
|
else {
|
|
previousMediaCount = account.mediaCount || 0;
|
|
logger.info(`Retrying @${cleanUsername} from cursor (${previousMediaCount} items)...`);
|
|
}
|
|
}
|
|
setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId
|
|
? {
|
|
...acc,
|
|
status: "fetching" as const,
|
|
elapsedTime: 0,
|
|
remainingTime: null,
|
|
error: undefined,
|
|
showDiff: false,
|
|
mediaCount: previousMediaCount,
|
|
previousMediaCount: previousMediaCount,
|
|
}
|
|
: acc));
|
|
accountStopFlagsRef.current.set(accountId, false);
|
|
const settings = getSettings();
|
|
const timeoutSeconds = settings.fetchTimeout || 60;
|
|
accountStartTimesRef.current.set(accountId, Date.now());
|
|
accountTimeoutSecondsRef.current.set(accountId, timeoutSeconds);
|
|
setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId
|
|
? { ...acc, elapsedTime: 0, remainingTime: timeoutSeconds }
|
|
: acc));
|
|
const timerInterval = window.setInterval(() => {
|
|
const startTime = accountStartTimesRef.current.get(accountId);
|
|
const timeoutSecs = accountTimeoutSecondsRef.current.get(accountId) || timeoutSeconds;
|
|
if (startTime) {
|
|
const elapsed = Math.floor((Date.now() - startTime) / 1000);
|
|
const remaining = Math.max(0, timeoutSecs - elapsed);
|
|
setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId
|
|
? { ...acc, elapsedTime: elapsed, remainingTime: remaining }
|
|
: acc));
|
|
if (remaining <= 0) {
|
|
accountStopFlagsRef.current.set(accountId, true);
|
|
clearInterval(timerInterval);
|
|
accountTimersRef.current.delete(accountId);
|
|
accountStartTimesRef.current.delete(accountId);
|
|
accountTimeoutSecondsRef.current.delete(accountId);
|
|
const mediaCount = accountMediaCountRef.current.get(accountId) || 0;
|
|
setMultipleAccounts((prev) => prev.map((acc) => {
|
|
if (acc.id === accountId) {
|
|
const status = mediaCount === 0 ? ("failed" as const) : ("incomplete" as const);
|
|
return { ...acc, status, remainingTime: 0, mediaCount };
|
|
}
|
|
return acc;
|
|
}));
|
|
CleanupExtractorProcesses().catch(() => { });
|
|
}
|
|
}
|
|
}, 1000);
|
|
accountTimersRef.current.set(accountId, timerInterval);
|
|
try {
|
|
let hasMore = true;
|
|
let page = 0;
|
|
while (hasMore && !stopAllRef.current) {
|
|
if (accountStopFlagsRef.current.get(accountId)) {
|
|
break;
|
|
}
|
|
const response = await ExtractTimeline({
|
|
username: cleanUsername,
|
|
auth_token: authToken.trim(),
|
|
timeline_type: "timeline",
|
|
batch_size: batchSizeRetry,
|
|
page: page,
|
|
media_type: fetchedMediaType || "all",
|
|
retweets: false,
|
|
cursor: cursor,
|
|
});
|
|
const data: TwitterResponse = JSON.parse(response);
|
|
if (!accountInfo && data.account_info) {
|
|
accountInfo = data.account_info;
|
|
}
|
|
allTimeline = mergeTimelines(allTimeline, data.timeline);
|
|
cursor = data.cursor;
|
|
hasMore = !!data.cursor && !data.completed;
|
|
page++;
|
|
if (cursor) {
|
|
saveCursor(cleanUsername, cursor);
|
|
}
|
|
const currentMediaCount = allTimeline.length;
|
|
accountMediaCountRef.current.set(accountId, currentMediaCount);
|
|
const hasNewItems = currentMediaCount > previousMediaCount;
|
|
setMultipleAccounts((prev) => prev.map((acc) => {
|
|
if (acc.id === accountId) {
|
|
return {
|
|
...acc,
|
|
accountInfo: accountInfo || undefined,
|
|
previousMediaCount: previousMediaCount,
|
|
mediaCount: currentMediaCount,
|
|
showDiff: hasNewItems,
|
|
cursor: cursor,
|
|
};
|
|
}
|
|
return acc;
|
|
}));
|
|
if (page % 3 === 0 || !hasMore || accountStopFlagsRef.current.get(accountId)) {
|
|
saveFetchState({
|
|
username: cleanUsername,
|
|
cursor: cursor || "",
|
|
timeline: allTimeline,
|
|
accountInfo: accountInfo,
|
|
totalFetched: allTimeline.length,
|
|
completed: !hasMore,
|
|
authToken: authToken.trim(),
|
|
mediaType: fetchedMediaType || "all",
|
|
retweets: false,
|
|
timelineType: "timeline",
|
|
});
|
|
}
|
|
if (hasNewItems) {
|
|
setTimeout(() => {
|
|
setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId ? { ...acc, showDiff: false } : acc));
|
|
}, 1000);
|
|
}
|
|
previousMediaCount = currentMediaCount;
|
|
if (accountInfo) {
|
|
try {
|
|
await SaveAccountToDBWithStatus(accountInfo.name, accountInfo.nick, accountInfo.profile_image, allTimeline.length, JSON.stringify({
|
|
account_info: accountInfo,
|
|
timeline: allTimeline,
|
|
total_urls: allTimeline.length,
|
|
metadata: {
|
|
new_entries: data.timeline.length,
|
|
page: page,
|
|
batch_size: batchSizeRetry,
|
|
has_more: hasMore,
|
|
cursor: cursor,
|
|
completed: !hasMore,
|
|
},
|
|
cursor: cursor,
|
|
completed: !hasMore,
|
|
}), fetchedMediaType || "all", cursor || "", !hasMore);
|
|
}
|
|
catch (err) {
|
|
reportDatabaseSaveError("retry progress", err);
|
|
}
|
|
}
|
|
}
|
|
const startTime = accountStartTimesRef.current.get(accountId);
|
|
const elapsedSecs = startTime ? Math.floor((Date.now() - startTime) / 1000) : 0;
|
|
clearInterval(timerInterval);
|
|
accountTimersRef.current.delete(accountId);
|
|
accountStartTimesRef.current.delete(accountId);
|
|
accountTimeoutSecondsRef.current.delete(accountId);
|
|
const finalMediaCount = allTimeline.length;
|
|
accountMediaCountRef.current.set(accountId, finalMediaCount);
|
|
const wasTimeout = accountStopFlagsRef.current.get(accountId);
|
|
if (wasTimeout) {
|
|
logger.warning(`@${account.username}: timeout - ${finalMediaCount} items (${elapsedSecs}s)`);
|
|
setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId
|
|
? { ...acc, status: finalMediaCount === 0 ? ("failed" as const) : ("incomplete" as const) }
|
|
: acc));
|
|
}
|
|
else if (stopAllRef.current) {
|
|
logger.info(`@${account.username}: stopped - ${finalMediaCount} items (${elapsedSecs}s)`);
|
|
setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId
|
|
? { ...acc, status: finalMediaCount === 0 ? ("failed" as const) : ("incomplete" as const) }
|
|
: acc));
|
|
}
|
|
else if (hasMore) {
|
|
logger.warning(`@${account.username}: incomplete - ${finalMediaCount} items (${elapsedSecs}s)`);
|
|
setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId
|
|
? { ...acc, status: finalMediaCount === 0 ? ("failed" as const) : ("incomplete" as const) }
|
|
: acc));
|
|
}
|
|
else {
|
|
logger.success(`@${account.username}: completed - ${finalMediaCount} items (${elapsedSecs}s)`);
|
|
clearFetchState(cleanUsername);
|
|
clearCursor(cleanUsername);
|
|
setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId ? { ...acc, status: "completed" as const, cursor: undefined } : acc));
|
|
}
|
|
}
|
|
catch (error) {
|
|
const startTime = accountStartTimesRef.current.get(accountId);
|
|
const elapsedSecs = startTime ? Math.floor((Date.now() - startTime) / 1000) : 0;
|
|
const timerInterval = accountTimersRef.current.get(accountId);
|
|
if (timerInterval) {
|
|
clearInterval(timerInterval);
|
|
accountTimersRef.current.delete(accountId);
|
|
}
|
|
accountStartTimesRef.current.delete(accountId);
|
|
accountTimeoutSecondsRef.current.delete(accountId);
|
|
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
const mediaCount = accountMediaCountRef.current.get(accountId) || 0;
|
|
const isAuthError = errorMsg.toLowerCase().includes("401") ||
|
|
errorMsg.toLowerCase().includes("unauthorized") ||
|
|
errorMsg.toLowerCase().includes("auth token may be invalid") ||
|
|
errorMsg.toLowerCase().includes("auth token may be expired") ||
|
|
errorMsg.toLowerCase().includes("invalid or expired");
|
|
const status = isAuthError || mediaCount === 0 ? ("failed" as const) : ("incomplete" as const);
|
|
accountMediaCountRef.current.delete(accountId);
|
|
setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId
|
|
? {
|
|
...acc,
|
|
status,
|
|
error: errorMsg,
|
|
mediaCount,
|
|
}
|
|
: acc));
|
|
logger.error(`@${account.username}: failed - ${errorMsg} (${elapsedSecs}s)`);
|
|
}
|
|
};
|
|
const renderPage = () => {
|
|
switch (currentPage) {
|
|
case "settings":
|
|
return <SettingsPage />;
|
|
case "debug":
|
|
return <DebugLoggerPage />;
|
|
case "database":
|
|
return (<DatabaseView onBack={() => setCurrentPage("main")} onLoadAccount={handleLoadFromDB} onUpdateSelected={handleUpdateSelected}/>);
|
|
default:
|
|
return (<>
|
|
<Header version={CURRENT_VERSION} hasUpdate={hasUpdate} releaseDate={releaseDate}/>
|
|
|
|
<SearchBar username={username} loading={loading} onUsernameChange={setUsername} onFetch={handleFetch} onStopFetch={handleStopFetch} onResume={handleResume} onClearResume={handleClearResume} resumeInfo={resumeInfo} history={fetchHistory} onHistorySelect={handleHistorySelect} onHistoryRemove={removeFromHistory} hasResult={!!result} elapsedTime={elapsedTime} remainingTime={remainingTime} fetchType={fetchType} onFetchTypeChange={handleFetchTypeChange} multipleAccounts={multipleAccounts} onImportFile={handleImportFile} onFetchAll={handleFetchAll} onStopAll={handleStopAll} onStopAccount={handleStopAccount} onRetryAccount={handleRetryAccount} isFetchingAll={isFetchingAll} mode={searchMode} privateType={searchPrivateType} onModeChange={(mode, privateType) => {
|
|
setSearchMode(mode);
|
|
if (privateType) {
|
|
setSearchPrivateType(privateType);
|
|
}
|
|
}}/>
|
|
|
|
{result && fetchType === "single" && (<div className="mt-4">
|
|
<MediaList accountInfo={result.account_info} timeline={result.timeline} totalUrls={result.total_urls} fetchedMediaType={fetchedMediaType} newMediaCount={newMediaCount}/>
|
|
</div>)}
|
|
</>);
|
|
}
|
|
};
|
|
return (<TooltipProvider>
|
|
<div className="min-h-screen bg-background flex flex-col">
|
|
<DependencySetupDialog />
|
|
<TitleBar />
|
|
<Sidebar currentPage={currentPage} onPageChange={setCurrentPage}/>
|
|
|
|
|
|
<div className="flex-1 ml-14 mt-10 p-4 md:p-8">
|
|
<div className="max-w-5xl mx-auto">
|
|
{renderPage()}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</TooltipProvider>);
|
|
}
|
|
export default App;
|