diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index e6b7917..75fe6a9 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,2 +1 @@ -github: afkarxyz -ko_fi: afkarxyz +ko_fi: afkarxyz \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..180a5d8 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,344 @@ +name: Build Multi-Platform + +on: + push: + tags: + - 'v*' + workflow_dispatch: + +env: + NODE_VERSION: '24' + PNPM_VERSION: '10' + APP_NAME: TwitterXMediaBatchDownloader + +jobs: + build-windows: + name: Build Windows + runs-on: windows-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Install pnpm + uses: pnpm/action-setup@v4 + with: + version: ${{ env.PNPM_VERSION }} + + - name: Get pnpm store directory + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + - name: Setup pnpm cache + continue-on-error: true + uses: actions/cache@v4 + with: + path: ${{ env.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('frontend/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- + + - name: Install Wails CLI + run: go install github.com/wailsapp/wails/v2/cmd/wails@latest + + - name: Install frontend dependencies + working-directory: frontend + run: | + pnpm install + pnpm run generate-icon + + - name: Install UPX + run: choco install upx -y + + - name: Build application + run: wails build -platform windows/amd64 + + - name: Compress with UPX + run: upx --best --lzma "build\bin\${{ env.APP_NAME }}.exe" + + - name: Prepare artifacts + shell: pwsh + run: | + New-Item -ItemType Directory -Force -Path dist | Out-Null + Copy-Item -Path "build\bin\${{ env.APP_NAME }}.exe" -Destination "dist\${{ env.APP_NAME }}.exe" + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: windows-portable + path: dist/${{ env.APP_NAME }}.exe + retention-days: 7 + + build-macos: + name: Build macOS + runs-on: macos-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Install pnpm + uses: pnpm/action-setup@v4 + with: + version: ${{ env.PNPM_VERSION }} + + - name: Get pnpm store directory + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + - name: Setup pnpm cache + continue-on-error: true + uses: actions/cache@v4 + with: + path: ${{ env.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('frontend/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- + + - name: Install Wails CLI + run: go install github.com/wailsapp/wails/v2/cmd/wails@latest + + - name: Install frontend dependencies + working-directory: frontend + run: | + pnpm install + pnpm run generate-icon + + - name: Build application + run: wails build -platform darwin/universal + + - name: Create DMG + run: | + mkdir -p dist + brew install create-dmg || true + + create-dmg \ + --volname "${{ env.APP_NAME }}" \ + --window-pos 200 120 \ + --window-size 600 400 \ + --icon-size 100 \ + --icon "${{ env.APP_NAME }}.app" 175 120 \ + --hide-extension "${{ env.APP_NAME }}.app" \ + --app-drop-link 425 120 \ + "dist/${{ env.APP_NAME }}.dmg" \ + "build/bin/${{ env.APP_NAME }}.app" || \ + hdiutil create -volname "${{ env.APP_NAME }}" -srcfolder "build/bin/${{ env.APP_NAME }}.app" -ov -format UDZO "dist/${{ env.APP_NAME }}.dmg" + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: macos-portable + path: dist/${{ env.APP_NAME }}.dmg + retention-days: 7 + + build-linux: + name: Build Linux + runs-on: ubuntu-24.04 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Install pnpm + uses: pnpm/action-setup@v4 + with: + version: ${{ env.PNPM_VERSION }} + + - name: Get pnpm store directory + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + - name: Setup pnpm cache + continue-on-error: true + uses: actions/cache@v4 + with: + path: ${{ env.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('frontend/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y libgtk-3-dev libwebkit2gtk-4.1-dev libfuse2 imagemagick upx-ucl + sudo ln -sf /usr/lib/x86_64-linux-gnu/pkgconfig/webkit2gtk-4.1.pc /usr/lib/x86_64-linux-gnu/pkgconfig/webkit2gtk-4.0.pc + + - name: Install Wails CLI + run: go install github.com/wailsapp/wails/v2/cmd/wails@latest + + - name: Install frontend dependencies + working-directory: frontend + run: | + pnpm install + pnpm run generate-icon + + - name: Build application + run: wails build -platform linux/amd64 + + - name: Compress with UPX + run: upx --best --lzma "build/bin/${{ env.APP_NAME }}" + + - name: Cache appimagetool + id: cache-appimagetool + uses: actions/cache@v4 + with: + path: appimagetool + key: appimagetool-x86_64-v2 + + - name: Download appimagetool + if: steps.cache-appimagetool.outputs.cache-hit != 'true' + run: | + wget --timeout=30 --tries=5 --retry-connrefused --waitretry=5 -O appimagetool "https://github.com/AppImage/appimagetool/releases/download/continuous/appimagetool-x86_64.AppImage" || \ + wget --timeout=30 --tries=5 --retry-connrefused --waitretry=5 -O appimagetool "https://github.com/AppImage/appimagetool/releases/download/1.9.1/appimagetool-x86_64.AppImage" + + - name: Make appimagetool executable + run: chmod +x appimagetool + + - name: Create AppImage + run: | + mkdir -p AppDir/usr/bin + mkdir -p AppDir/usr/share/applications + mkdir -p AppDir/usr/share/icons/hicolor/256x256/apps + + cp "build/bin/${{ env.APP_NAME }}" AppDir/usr/bin/ + + cat > AppDir/twitterxmediabatchdownloader.desktop << 'EOF' + [Desktop Entry] + Name=Twitter/X Media Batch Downloader + Exec=TwitterXMediaBatchDownloader + Icon=twitterxmediabatchdownloader + Type=Application + Categories=Network;Utility; + Comment=Download Twitter/X media in batches + EOF + + cp AppDir/twitterxmediabatchdownloader.desktop AppDir/usr/share/applications/ + + if [ -f "build/appicon.png" ]; then + convert build/appicon.png -resize 256x256 AppDir/twitterxmediabatchdownloader.png + elif [ -f "frontend/public/icon.svg" ]; then + convert -background none -size 256x256 frontend/public/icon.svg AppDir/twitterxmediabatchdownloader.png + else + echo "Warning: No icon found, building without icon" + fi + + if [ -f "AppDir/twitterxmediabatchdownloader.png" ]; then + cp AppDir/twitterxmediabatchdownloader.png AppDir/usr/share/icons/hicolor/256x256/apps/ + cp AppDir/twitterxmediabatchdownloader.png AppDir/.DirIcon + fi + + cat > AppDir/AppRun << 'EOF' + #!/bin/sh + SELF=$(readlink -f "$0") + HERE=${SELF%/*} + export PATH="${HERE}/usr/bin/:${PATH}" + export LD_LIBRARY_PATH="${HERE}/usr/lib/:${LD_LIBRARY_PATH}" + exec "${HERE}/usr/bin/TwitterXMediaBatchDownloader" "$@" + EOF + chmod +x AppDir/AppRun + + mkdir -p dist + ARCH=x86_64 ./appimagetool --no-appstream AppDir "dist/${{ env.APP_NAME }}.AppImage" + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: linux-portable + path: dist/${{ env.APP_NAME }}.AppImage + retention-days: 7 + + create-release: + name: Create Release + needs: [build-windows, build-macos, build-linux] + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/v') + permissions: + contents: write + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Display structure of downloaded files + run: ls -R artifacts + + - name: Create Release + uses: softprops/action-gh-release@v2 + with: + draft: true + prerelease: false + generate_release_notes: false + body: | + ## Changelog + + ## Downloads + + - `TwitterXMediaBatchDownloader.exe` - Windows + - `TwitterXMediaBatchDownloader.dmg` - macOS + - `TwitterXMediaBatchDownloader.AppImage` - Linux + +
+ Linux Requirements + + The AppImage requires `webkit2gtk-4.1` to be installed on your system: + + **Ubuntu/Debian:** + ```bash + sudo apt install libwebkit2gtk-4.1-0 + ``` + + **Arch Linux:** + ```bash + sudo pacman -S webkit2gtk-4.1 + ``` + + **Fedora:** + ```bash + sudo dnf install webkit2gtk4.1 + ``` + + After installing the dependency, make the AppImage executable: + ```bash + chmod +x TwitterXMediaBatchDownloader.AppImage + ./TwitterXMediaBatchDownloader.AppImage + ``` + +
+ files: | + artifacts/windows-portable/${{ env.APP_NAME }}.exe + artifacts/macos-portable/${{ env.APP_NAME }}.dmg + artifacts/linux-portable/${{ env.APP_NAME }}.AppImage + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore deleted file mode 100644 index c2cfdc7..0000000 --- a/.gitignore +++ /dev/null @@ -1,62 +0,0 @@ -# Wails Build -build/ -backend/bin/ -*.exe -*.dll -*.dylib -*.so - -# Wails Generated Files -frontend/wailsjs/ - -# Go -*.test -*.out -go.work -go.work.sum - -# Node / Frontend -node_modules/ -frontend/node_modules/ -frontend/dist/ -frontend/.vite/ -npm-debug.log* -yarn-debug.log* -yarn-error.log* -pnpm-debug.log* -.pnpm-store/ -.npm -.yarn -*.tsbuildinfo - -# IDE / Editors -.vscode/ -.idea/ -*.swp -*.swo -*~ -.DS_Store - -# OS -Thumbs.db -desktop.ini - -# Environment -.env -.env.local -.env.*.local - -# Logs -*.log -logs/ - -# Temporary files -tmp/ -temp/ -*.tmp -*.bak -*.old - -# Build notes (optional - uncomment if you don't want to commit) -# BUILD_NOTES.md -build.txt \ No newline at end of file diff --git a/README.md b/README.md index 16f3dd7..c9cc771 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,4 @@ -[![GitHub All Releases](https://img.shields.io/github/downloads/afkarxyz/Twitter-X-Media-Batch-Downloader/total?style=for-the-badge)](https://github.com/afkarxyz/Twitter-X-Media-Batch-Downloader/releases) - -![TwitterXMediaBatchDownloader](https://github.com/user-attachments/assets/354d7470-c01c-4aa6-9da1-ea6c42d27330) - -
+# Twitter/X Media Batch Downloader A GUI tool to download original-quality images and videos from Twitter/X accounts, powered by gallery-dl by @mikf @@ -10,18 +6,14 @@ A GUI tool to download original-quality images and videos from Twitter/X account ![macOS](https://img.shields.io/badge/macOS-10.13%2B-000000?style=for-the-badge&logo=apple&logoColor=white) ![Linux](https://img.shields.io/badge/Linux-Any-FCC624?style=for-the-badge&logo=linux&logoColor=white) -
- ### [Download](https://github.com/afkarxyz/Twitter-X-Media-Batch-Downloader/releases) +![Image](https://github.com/user-attachments/assets/a3ade39c-d611-49ce-81bb-121f30488ae2) + - If you're familiar with userscripts, install it [**here**](https://greasyfork.org/en/scripts/551198) - To install the Chrome extension version, click [**here**](https://chromewebstore.google.com/detail/aboccmnimgpjehlplnanplbhlccohhhn) -## Screenshot - -![Image](https://github.com/user-attachments/assets/a3ade39c-d611-49ce-81bb-121f30488ae2) - ## How to Obtain Auth Token > [!Warning] @@ -40,5 +32,3 @@ A GUI tool to download original-quality images and videos from Twitter/X account 6. Do not log out of your account, as a new `auth_token` will be generated by Twitter ![image](https://github.com/user-attachments/assets/8e81dd8f-f8be-4254-9cf6-cacfa97743e9) - -[![Ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/afkarxyz) diff --git a/app.go b/app.go index 2f2d64b..d4afc7a 100644 --- a/app.go +++ b/app.go @@ -10,42 +10,36 @@ import ( "github.com/wailsapp/wails/v2/pkg/runtime" ) -// App struct type App struct { ctx context.Context downloadCtx context.Context downloadCancel context.CancelFunc } -// NewApp creates a new App application struct func NewApp() *App { return &App{} } -// startup is called when the app starts. The context is saved -// so we can call the runtime methods func (a *App) startup(ctx context.Context) { a.ctx = ctx - // Initialize database - backend.InitDB() - // Kill any leftover extractor processes from previous session + + if err := backend.InitDB(); err != nil { + runtime.LogErrorf(ctx, "failed to initialize database: %v", err) + } + backend.KillAllExtractorProcesses() } -// shutdown is called when the app is closing func (a *App) shutdown(ctx context.Context) { backend.CloseDB() - // Kill any running extractor processes + backend.KillAllExtractorProcesses() } -// CleanupExtractorProcesses kills all running extractor processes -// Can be called from frontend when user wants to stop/reset func (a *App) CleanupExtractorProcesses() { backend.KillAllExtractorProcesses() } -// TimelineRequest represents the request structure for timeline extraction type TimelineRequest struct { Username string `json:"username"` AuthToken string `json:"auth_token"` @@ -54,10 +48,9 @@ type TimelineRequest struct { Page int `json:"page"` MediaType string `json:"media_type"` Retweets bool `json:"retweets"` - Cursor string `json:"cursor,omitempty"` // Resume from this cursor position + Cursor string `json:"cursor,omitempty"` } -// DateRangeRequest represents the request structure for date range extraction type DateRangeRequest struct { Username string `json:"username"` AuthToken string `json:"auth_token"` @@ -67,9 +60,8 @@ type DateRangeRequest struct { Retweets bool `json:"retweets"` } -// ExtractTimeline extracts media from user timeline func (a *App) ExtractTimeline(req TimelineRequest) (string, error) { - // Username not required for bookmarks only + if req.Username == "" && req.TimelineType != "bookmarks" { return "", fmt.Errorf("username is required") } @@ -101,7 +93,6 @@ func (a *App) ExtractTimeline(req TimelineRequest) (string, error) { return string(jsonData), nil } -// ExtractDateRange extracts media based on date range func (a *App) ExtractDateRange(req DateRangeRequest) (string, error) { if req.Username == "" { return "", fmt.Errorf("username is required") @@ -138,13 +129,11 @@ func (a *App) ExtractDateRange(req DateRangeRequest) (string, error) { return string(jsonData), nil } -// OpenFolder opens a folder in the file explorer func (a *App) OpenFolder(path string) error { if path == "" { return fmt.Errorf("path is required") } - // Clean the path to use correct separators for the OS cleanPath := filepath.Clean(path) err := backend.OpenFolderInExplorer(cleanPath) @@ -155,51 +144,52 @@ func (a *App) OpenFolder(path string) error { return nil } -// SelectFolder opens a folder selection dialog and returns the selected path func (a *App) SelectFolder(defaultPath string) (string, error) { return backend.SelectFolderDialog(a.ctx, defaultPath) } -// GetDefaults returns the default configuration func (a *App) GetDefaults() map[string]string { return map[string]string{ "downloadPath": backend.GetDefaultDownloadPath(), } } -// Quit closes the application +func (a *App) IsExtractorInstalled() bool { + return backend.IsExtractorInstalled() +} + +func (a *App) DownloadExtractor() error { + return backend.DownloadExtractor(nil) +} + func (a *App) Quit() { panic("quit") } -// DownloadMediaRequest represents the request for downloading media (legacy) type DownloadMediaRequest struct { URLs []string `json:"urls"` OutputDir string `json:"output_dir"` Username string `json:"username"` - Proxy string `json:"proxy,omitempty"` // Optional proxy URL (e.g., http://proxy:port or socks5://proxy:port) + Proxy string `json:"proxy,omitempty"` } -// MediaItemRequest represents a media item with metadata type MediaItemRequest struct { URL string `json:"url"` Date string `json:"date"` TweetID backend.TweetIDString `json:"tweet_id"` Type string `json:"type"` - Content string `json:"content,omitempty"` // Tweet text content (for text-only tweets) - OriginalFilename string `json:"original_filename,omitempty"` // Original filename from API - AuthorUsername string `json:"author_username,omitempty"` // Username of tweet author (for bookmarks and likes) + Content string `json:"content,omitempty"` + OriginalFilename string `json:"original_filename,omitempty"` + AuthorUsername string `json:"author_username,omitempty"` } -// DownloadMediaWithMetadataRequest represents the request for downloading media with metadata type DownloadMediaWithMetadataRequest struct { Items []MediaItemRequest `json:"items"` OutputDir string `json:"output_dir"` Username string `json:"username"` - Proxy string `json:"proxy,omitempty"` // Optional proxy URL (e.g., http://proxy:port or socks5://proxy:port) + Proxy string `json:"proxy,omitempty"` } -// DownloadMediaResponse represents the response for download operation type DownloadMediaResponse struct { Success bool `json:"success"` Downloaded int `json:"downloaded"` @@ -208,7 +198,6 @@ type DownloadMediaResponse struct { Message string `json:"message"` } -// DownloadMedia downloads media files from URLs (legacy) func (a *App) DownloadMedia(req DownloadMediaRequest) (DownloadMediaResponse, error) { if len(req.URLs) == 0 { return DownloadMediaResponse{ @@ -222,7 +211,6 @@ func (a *App) DownloadMedia(req DownloadMediaRequest) (DownloadMediaResponse, er outputDir = backend.GetDefaultDownloadPath() } - // Create subfolder for username if provided if req.Username != "" { outputDir = filepath.Join(outputDir, req.Username) } @@ -247,21 +235,18 @@ func (a *App) DownloadMedia(req DownloadMediaRequest) (DownloadMediaResponse, er }, nil } -// DownloadProgress represents download progress event data type DownloadProgress struct { Current int `json:"current"` Total int `json:"total"` Percent int `json:"percent"` } -// DownloadItemStatus represents per-item download status event data type DownloadItemStatus struct { TweetID int64 `json:"tweet_id"` Index int `json:"index"` - Status string `json:"status"` // "success", "failed", "skipped" + Status string `json:"status"` } -// DownloadMediaWithMetadata downloads media files with proper naming and categorization func (a *App) DownloadMediaWithMetadata(req DownloadMediaWithMetadataRequest) (DownloadMediaResponse, error) { if len(req.Items) == 0 { return DownloadMediaResponse{ @@ -275,18 +260,15 @@ func (a *App) DownloadMediaWithMetadata(req DownloadMediaWithMetadataRequest) (D outputDir = backend.GetDefaultDownloadPath() } - // Convert request items to backend items - // For bookmarks and likes, use author_username from each item if available items := make([]backend.MediaItem, len(req.Items)) for i, item := range req.Items { - // Use original filename from API if available, otherwise extract from URL + originalFilename := item.OriginalFilename if originalFilename == "" { - // Fallback: extract from URL if not provided in API response + originalFilename = backend.ExtractOriginalFilename(item.URL) } - // For bookmarks and likes, use author_username from item, otherwise use req.Username username := req.Username if item.AuthorUsername != "" { username = item.AuthorUsername @@ -303,10 +285,8 @@ func (a *App) DownloadMediaWithMetadata(req DownloadMediaWithMetadataRequest) (D } } - // Create cancellable context a.downloadCtx, a.downloadCancel = context.WithCancel(context.Background()) - // Progress callback progressCallback := func(current, total int) { percent := 0 if total > 0 { @@ -319,7 +299,6 @@ func (a *App) DownloadMediaWithMetadata(req DownloadMediaWithMetadataRequest) (D }) } - // Per-item status callback itemStatusCallback := func(tweetID int64, index int, status string) { runtime.EventsEmit(a.ctx, "download-item-status", DownloadItemStatus{ TweetID: tweetID, @@ -339,7 +318,6 @@ func (a *App) DownloadMediaWithMetadata(req DownloadMediaWithMetadataRequest) (D }, err } - // Clear cancel function a.downloadCancel = nil return DownloadMediaResponse{ @@ -351,7 +329,6 @@ func (a *App) DownloadMediaWithMetadata(req DownloadMediaWithMetadataRequest) (D }, nil } -// StopDownload cancels the current download operation func (a *App) StopDownload() bool { if a.downloadCancel != nil { a.downloadCancel() @@ -361,24 +338,18 @@ func (a *App) StopDownload() bool { return false } -// Database functions - -// SaveAccountToDB saves account data to database func (a *App) SaveAccountToDB(username, name, profileImage string, totalMedia int, responseJSON string, mediaType string) error { return backend.SaveAccount(username, name, profileImage, totalMedia, responseJSON, mediaType) } -// SaveAccountToDBWithStatus saves account data with cursor and completion status for resume capability func (a *App) SaveAccountToDBWithStatus(username, name, profileImage string, totalMedia int, responseJSON string, mediaType string, cursor string, completed bool) error { return backend.SaveAccountWithStatus(username, name, profileImage, totalMedia, responseJSON, mediaType, cursor, completed) } -// GetAllAccountsFromDB returns all saved accounts func (a *App) GetAllAccountsFromDB() ([]backend.AccountListItem, error) { return backend.GetAllAccounts() } -// GetAccountFromDB returns account data by ID func (a *App) GetAccountFromDB(id int64) (string, error) { acc, err := backend.GetAccountByID(id) if err != nil { @@ -387,67 +358,53 @@ func (a *App) GetAccountFromDB(id int64) (string, error) { return acc.ResponseJSON, nil } -// DeleteAccountFromDB deletes an account from database func (a *App) DeleteAccountFromDB(id int64) error { return backend.DeleteAccount(id) } -// ClearAllAccountsFromDB deletes all accounts from database func (a *App) ClearAllAccountsFromDB() error { return backend.ClearAllAccounts() } -// ExportAccountJSON exports account to JSON file in specified directory func (a *App) ExportAccountJSON(id int64, outputDir string) (string, error) { return backend.ExportAccountToFile(id, outputDir) } -// ExportAccountsTXT exports selected accounts to TXT file in specified directory func (a *App) ExportAccountsTXT(ids []int64, outputDir string) (string, error) { return backend.ExportAccountsToTXT(ids, outputDir) } -// UpdateAccountGroup updates the group for an account func (a *App) UpdateAccountGroup(id int64, groupName, groupColor string) error { return backend.UpdateAccountGroup(id, groupName, groupColor) } -// GetAllGroups returns all unique groups func (a *App) GetAllGroups() ([]map[string]string, error) { return backend.GetAllGroups() } -// FFmpeg functions - -// IsFFmpegInstalled checks if ffmpeg is available func (a *App) IsFFmpegInstalled() bool { return backend.IsFFmpegInstalled() } -// DownloadFFmpeg downloads ffmpeg binary func (a *App) DownloadFFmpeg() error { return backend.DownloadFFmpeg(nil) } -// IsExifToolInstalled checks if exiftool is available func (a *App) IsExifToolInstalled() bool { return backend.IsExifToolInstalled() } -// DownloadExifTool downloads exiftool binary func (a *App) DownloadExifTool() error { return backend.DownloadExifTool(nil) } -// ConvertGIFsRequest represents request for converting GIFs type ConvertGIFsRequest struct { FolderPath string `json:"folder_path"` - Quality string `json:"quality"` // "fast" or "better" - Resolution string `json:"resolution"` // "original", "high", "medium", "low" + Quality string `json:"quality"` + Resolution string `json:"resolution"` DeleteOriginal bool `json:"delete_original"` } -// ConvertGIFsResponse represents response for GIF conversion type ConvertGIFsResponse struct { Success bool `json:"success"` Converted int `json:"converted"` @@ -455,7 +412,6 @@ type ConvertGIFsResponse struct { Message string `json:"message"` } -// ConvertGIFs converts MP4 files in gifs folder to actual GIF format func (a *App) ConvertGIFs(req ConvertGIFsRequest) (ConvertGIFsResponse, error) { if !backend.IsFFmpegInstalled() { return ConvertGIFsResponse{ @@ -464,7 +420,6 @@ func (a *App) ConvertGIFs(req ConvertGIFsRequest) (ConvertGIFsResponse, error) { }, nil } - // Default values if not provided quality := req.Quality if quality == "" { quality = "fast" @@ -490,41 +445,34 @@ func (a *App) ConvertGIFs(req ConvertGIFsRequest) (ConvertGIFsResponse, error) { }, nil } -// ImportAccountResponse represents the response for import operation type ImportAccountResponse struct { Success bool `json:"success"` Username string `json:"username"` Message string `json:"message"` } -// CheckFolderExists checks if a folder exists for the given username func (a *App) CheckFolderExists(basePath, username string) bool { return backend.CheckFolderExists(basePath, username) } -// CheckGifsFolderExists checks if a gifs subfolder exists for the given username func (a *App) CheckGifsFolderExists(basePath, username string) bool { return backend.CheckGifsFolderExists(basePath, username) } -// CheckGifsFolderHasMP4 checks if the gifs folder has any MP4 files to convert func (a *App) CheckGifsFolderHasMP4(basePath, username string) bool { return backend.CheckGifsFolderHasMP4(basePath, username) } -// GetFolderPath returns the full path for a username folder func (a *App) GetFolderPath(basePath, username string) string { return backend.GetFolderPath(basePath, username) } -// GetGifsFolderPath returns the full path for a username's gifs folder func (a *App) GetGifsFolderPath(basePath, username string) string { return backend.GetGifsFolderPath(basePath, username) } -// ImportAccountFromJSON imports account from JSON file (supports both old and new format) func (a *App) ImportAccountFromJSON() (ImportAccountResponse, error) { - // Open file dialog + filePath, err := runtime.OpenFileDialog(a.ctx, runtime.OpenDialogOptions{ Title: "Import Account JSON", Filters: []runtime.FileFilter{ @@ -535,12 +483,10 @@ func (a *App) ImportAccountFromJSON() (ImportAccountResponse, error) { return ImportAccountResponse{Success: false, Message: err.Error()}, err } - // User cancelled if filePath == "" { return ImportAccountResponse{Success: false, Message: "Cancelled"}, nil } - // Import the file username, err := backend.ImportAccountFromFile(filePath) if err != nil { return ImportAccountResponse{Success: false, Message: err.Error()}, err diff --git a/backend/cmd_unix.go b/backend/cmd_unix.go index b2c613a..54359d9 100644 --- a/backend/cmd_unix.go +++ b/backend/cmd_unix.go @@ -4,7 +4,6 @@ package backend import "os/exec" -// hideWindow is a no-op on non-Windows platforms func hideWindow(cmd *exec.Cmd) { - // No action needed on Unix-like systems + } diff --git a/backend/cmd_windows.go b/backend/cmd_windows.go index 1cd228f..122d354 100644 --- a/backend/cmd_windows.go +++ b/backend/cmd_windows.go @@ -7,10 +7,9 @@ import ( "syscall" ) -// hideWindow sets the command to run without showing a console window on Windows func hideWindow(cmd *exec.Cmd) { cmd.SysProcAttr = &syscall.SysProcAttr{ HideWindow: true, - CreationFlags: 0x08000000, // CREATE_NO_WINDOW + CreationFlags: 0x08000000, } } diff --git a/backend/config.go b/backend/config.go index 878aa0c..f6fc8f3 100644 --- a/backend/config.go +++ b/backend/config.go @@ -10,21 +10,18 @@ import ( ) func GetDefaultDownloadPath() string { - // Get user's home directory + homeDir, err := os.UserHomeDir() if err != nil { - // Fallback to Public Pictures if can't get home dir + return "C:\\Users\\Public\\Pictures" } - // Return path to user's Pictures folder return filepath.Join(homeDir, "Pictures") } -// GetProxyURL gets proxy URL from environment variables or custom proxy setting -// Priority: customProxy > HTTP_PROXY/HTTPS_PROXY > http_proxy/https_proxy func GetProxyURL(customProxy string) (*url.URL, error) { - // First, check custom proxy setting + if customProxy != "" { proxyURL, err := url.Parse(customProxy) if err != nil { @@ -33,7 +30,6 @@ func GetProxyURL(customProxy string) (*url.URL, error) { return proxyURL, nil } - // Check environment variables (case-insensitive on Windows) proxyEnv := os.Getenv("HTTPS_PROXY") if proxyEnv == "" { proxyEnv = os.Getenv("https_proxy") @@ -53,10 +49,9 @@ func GetProxyURL(customProxy string) (*url.URL, error) { return proxyURL, nil } - return nil, nil // No proxy + return nil, nil } -// CreateHTTPClient creates an HTTP client with proxy support func CreateHTTPClient(customProxy string, timeout time.Duration) (*http.Client, error) { proxyURL, err := GetProxyURL(customProxy) if err != nil { diff --git a/backend/database.go b/backend/database.go index 4e3599e..ec2fcf0 100644 --- a/backend/database.go +++ b/backend/database.go @@ -8,10 +8,9 @@ import ( "path/filepath" "time" - _ "github.com/mattn/go-sqlite3" + _ "modernc.org/sqlite" ) -// AccountDB represents a saved account in the database type AccountDB struct { ID int64 `json:"id"` Username string `json:"username"` @@ -25,7 +24,6 @@ type AccountDB struct { Completed bool `json:"completed"` } -// AccountListItem represents a simplified account for listing type AccountListItem struct { ID int64 `json:"id"` Username string `json:"username"` @@ -44,7 +42,6 @@ type AccountListItem struct { var db *sql.DB -// GetDBPath returns the database file path func GetDBPath() string { homeDir, err := os.UserHomeDir() if err != nil { @@ -53,23 +50,24 @@ func GetDBPath() string { return filepath.Join(homeDir, ".twitterxmediabatchdownloader", "accounts.db") } -// InitDB initializes the database connection func InitDB() error { dbPath := GetDBPath() - // Create directory if not exists dir := filepath.Dir(dbPath) if err := os.MkdirAll(dir, 0755); err != nil { return err } var err error - db, err = sql.Open("sqlite3", dbPath) + db, err = sql.Open("sqlite", dbPath) if err != nil { return err } - // Create tables + if err := db.Ping(); err != nil { + return err + } + _, err = db.Exec(` CREATE TABLE IF NOT EXISTS accounts ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -91,33 +89,27 @@ func InitDB() error { return err } - // Add group columns if they don't exist (migration for existing databases) db.Exec("ALTER TABLE accounts ADD COLUMN group_name TEXT DEFAULT ''") db.Exec("ALTER TABLE accounts ADD COLUMN group_color TEXT DEFAULT ''") db.Exec("ALTER TABLE accounts ADD COLUMN media_type TEXT DEFAULT 'all'") db.Exec("ALTER TABLE accounts ADD COLUMN cursor TEXT DEFAULT ''") db.Exec("ALTER TABLE accounts ADD COLUMN completed INTEGER DEFAULT 1") - // Migration: Update unique constraint for existing databases - // This allows same username with different media types db.Exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_username_media_type ON accounts(username, media_type)") return nil } -// CloseDB closes the database connection func CloseDB() { if db != nil { db.Close() } } -// SaveAccount saves or updates an account in the database func SaveAccount(username, name, profileImage string, totalMedia int, responseJSON string, mediaType string) error { return SaveAccountWithStatus(username, name, profileImage, totalMedia, responseJSON, mediaType, "", true) } -// SaveAccountWithStatus saves or updates an account with cursor and completion status func SaveAccountWithStatus(username, name, profileImage string, totalMedia int, responseJSON string, mediaType string, cursor string, completed bool) error { if db == nil { if err := InitDB(); err != nil { @@ -125,7 +117,6 @@ func SaveAccountWithStatus(username, name, profileImage string, totalMedia int, } } - // Default to "all" if not specified if mediaType == "" { mediaType = "all" } @@ -151,7 +142,6 @@ func SaveAccountWithStatus(username, name, profileImage string, totalMedia int, return err } -// GetAllAccounts returns all saved accounts func GetAllAccounts() ([]AccountListItem, error) { if db == nil { if err := InitDB(); err != nil { @@ -185,7 +175,6 @@ func GetAllAccounts() ([]AccountListItem, error) { acc.LastFetched = lastFetched.Format("2006-01-02 15:04") acc.Completed = completedInt == 1 - // Extract followers_count and statuses_count from response_json if responseJSON != "" { var parsed map[string]interface{} if err := json.Unmarshal([]byte(responseJSON), &parsed); err == nil { @@ -206,7 +195,6 @@ func GetAllAccounts() ([]AccountListItem, error) { return accounts, nil } -// UpdateAccountGroup updates the group for an account func UpdateAccountGroup(id int64, groupName, groupColor string) error { if db == nil { if err := InitDB(); err != nil { @@ -218,7 +206,6 @@ func UpdateAccountGroup(id int64, groupName, groupColor string) error { return err } -// GetAllGroups returns all unique groups func GetAllGroups() ([]map[string]string, error) { if db == nil { if err := InitDB(); err != nil { @@ -249,7 +236,6 @@ func GetAllGroups() ([]map[string]string, error) { return groups, nil } -// ClearAllAccounts deletes all accounts from the database func ClearAllAccounts() error { if db == nil { if err := InitDB(); err != nil { @@ -261,7 +247,6 @@ func ClearAllAccounts() error { return err } -// GetAccountByID returns a specific account by ID func GetAccountByID(id int64) (*AccountDB, error) { if db == nil { if err := InitDB(); err != nil { @@ -284,7 +269,6 @@ func GetAccountByID(id int64) (*AccountDB, error) { acc.LastFetched = lastFetched acc.Completed = completedInt == 1 - // Convert legacy format if needed if converted, err := ConvertLegacyToNewFormat(acc.ResponseJSON); err == nil { acc.ResponseJSON = converted } @@ -292,7 +276,6 @@ func GetAccountByID(id int64) (*AccountDB, error) { return &acc, nil } -// DeleteAccount deletes an account from the database func DeleteAccount(id int64) error { if db == nil { if err := InitDB(); err != nil { @@ -304,7 +287,6 @@ func DeleteAccount(id int64) error { return err } -// LegacyMediaEntry represents media entry in old format type LegacyMediaEntry struct { TweetID string `json:"tweet_id"` URL string `json:"url"` @@ -312,7 +294,6 @@ type LegacyMediaEntry struct { Type string `json:"type"` } -// LegacyAccountFormat represents the old saved account format type LegacyAccountFormat struct { Username string `json:"username"` Nick string `json:"nick"` @@ -327,20 +308,17 @@ type LegacyAccountFormat struct { MediaList []LegacyMediaEntry `json:"media_list"` } -// ConvertLegacyToNewFormat converts old format to new TwitterResponse format func ConvertLegacyToNewFormat(jsonStr string) (string, error) { - // First check if it's already in new format (has account_info key) + var check map[string]interface{} if err := json.Unmarshal([]byte(jsonStr), &check); err != nil { return jsonStr, err } - // If already has account_info, it's new format - return as is if _, hasAccountInfo := check["account_info"]; hasAccountInfo { return jsonStr, nil } - // Check if it's legacy format (has username and media_list) if _, hasUsername := check["username"]; !hasUsername { return jsonStr, nil } @@ -348,13 +326,11 @@ func ConvertLegacyToNewFormat(jsonStr string) (string, error) { return jsonStr, nil } - // Parse as legacy format var legacy LegacyAccountFormat if err := json.Unmarshal([]byte(jsonStr), &legacy); err != nil { return jsonStr, err } - // Convert timeline entries timeline := make([]map[string]interface{}, len(legacy.MediaList)) for i, media := range legacy.MediaList { timeline[i] = map[string]interface{}{ @@ -366,7 +342,6 @@ func ConvertLegacyToNewFormat(jsonStr string) (string, error) { } } - // Build new format newFormat := map[string]interface{}{ "account_info": map[string]interface{}{ "name": legacy.Username, @@ -387,7 +362,6 @@ func ConvertLegacyToNewFormat(jsonStr string) (string, error) { }, } - // Convert back to JSON newJSON, err := json.Marshal(newFormat) if err != nil { return jsonStr, err @@ -396,20 +370,17 @@ func ConvertLegacyToNewFormat(jsonStr string) (string, error) { return string(newJSON), nil } -// ExportAccountToFile exports account JSON to a file func ExportAccountToFile(id int64, outputDir string) (string, error) { acc, err := GetAccountByID(id) if err != nil { return "", err } - // Create export directory if not exists exportDir := filepath.Join(outputDir, "twitterxmediabatchdownloader_backups") if err := os.MkdirAll(exportDir, 0755); err != nil { return "", err } - // Use username (nick) for filename filename := acc.Username if filename == "" { filename = acc.Name @@ -424,24 +395,21 @@ func ExportAccountToFile(id int64, outputDir string) (string, error) { return filePath, nil } -// ExportAccountsToTXT exports selected accounts to TXT file (one username per line) func ExportAccountsToTXT(ids []int64, outputDir string) (string, error) { if len(ids) == 0 { return "", fmt.Errorf("no accounts to export") } - // Create export directory if not exists exportDir := filepath.Join(outputDir, "twitterxmediabatchdownloader_backups") if err := os.MkdirAll(exportDir, 0755); err != nil { return "", err } - // Get all accounts by IDs var usernames []string for _, id := range ids { acc, err := GetAccountByID(id) if err != nil { - continue // Skip if account not found + continue } if acc.Username != "" { usernames = append(usernames, acc.Username) @@ -452,7 +420,6 @@ func ExportAccountsToTXT(ids []int64, outputDir string) (string, error) { return "", fmt.Errorf("no valid usernames found") } - // Create TXT content (one username per line) txtContent := "" for i, username := range usernames { if i > 0 { @@ -461,7 +428,6 @@ func ExportAccountsToTXT(ids []int64, outputDir string) (string, error) { txtContent += username } - // Filename: twitterxmediabatchdownloader_multiple.txt filePath := filepath.Join(exportDir, "twitterxmediabatchdownloader_multiple.txt") if err := os.WriteFile(filePath, []byte(txtContent), 0644); err != nil { @@ -471,7 +437,6 @@ func ExportAccountsToTXT(ids []int64, outputDir string) (string, error) { return filePath, nil } -// ImportAccountFromFile imports account from JSON file (supports both old and new format) func ImportAccountFromFile(filePath string) (string, error) { if db == nil { if err := InitDB(); err != nil { @@ -479,7 +444,6 @@ func ImportAccountFromFile(filePath string) (string, error) { } } - // Read file data, err := os.ReadFile(filePath) if err != nil { return "", err @@ -487,19 +451,16 @@ func ImportAccountFromFile(filePath string) (string, error) { jsonStr := string(data) - // Convert legacy format if needed convertedJSON, err := ConvertLegacyToNewFormat(jsonStr) if err != nil { return "", err } - // Parse the converted JSON to extract account info var parsed map[string]interface{} if err := json.Unmarshal([]byte(convertedJSON), &parsed); err != nil { return "", err } - // Extract account info accountInfo, ok := parsed["account_info"].(map[string]interface{}) if !ok { return "", fmt.Errorf("invalid JSON format: missing account_info") @@ -518,7 +479,6 @@ func ImportAccountFromFile(filePath string) (string, error) { return "", fmt.Errorf("invalid JSON format: missing username") } - // Save to database with default media type "all" for imported files err = SaveAccount(username, name, profileImage, totalURLs, convertedJSON, "all") if err != nil { return "", err diff --git a/backend/download.go b/backend/download.go index 9565d84..f1a74d6 100644 --- a/backend/download.go +++ b/backend/download.go @@ -15,32 +15,28 @@ import ( ) const ( - // MaxConcurrentDownloads is the number of parallel downloads MaxConcurrentDownloads = 10 ) -// MediaItem represents a media item with metadata for download type MediaItem struct { - URL string `json:"url"` - Date string `json:"date"` - TweetID int64 `json:"tweet_id"` - Type string `json:"type"` - Username string `json:"username"` - Content string `json:"content,omitempty"` // Tweet text content (for text-only tweets) - OriginalFilename string `json:"original_filename,omitempty"` // Original Twitter media filename (15 char alphanumeric) + URL string `json:"url"` + Date string `json:"date"` + TweetID int64 `json:"tweet_id"` + Type string `json:"type"` + Username string `json:"username"` + Content string `json:"content,omitempty"` + OriginalFilename string `json:"original_filename,omitempty"` } -// DownloadMediaFiles downloads media files from URLs to the output directory (legacy) func DownloadMediaFiles(urls []string, outputDir string, customProxy string) (downloaded int, failed int, err error) { - // Create output directory if it doesn't exist + if err := os.MkdirAll(outputDir, 0755); err != nil { return 0, len(urls), fmt.Errorf("failed to create output directory: %v", err) } - // Create HTTP client with proxy support client, err := CreateHTTPClient(customProxy, 60*time.Second) if err != nil { - // If proxy setup fails, use default client without proxy + client = &http.Client{ Timeout: 60 * time.Second, } @@ -50,7 +46,6 @@ func DownloadMediaFiles(urls []string, outputDir string, customProxy string) (do filename := extractFilename(mediaURL) outputPath := filepath.Join(outputDir, filename) - // Skip if file already exists if _, err := os.Stat(outputPath); err == nil { downloaded++ continue @@ -66,21 +61,16 @@ func DownloadMediaFiles(urls []string, outputDir string, customProxy string) (do return downloaded, failed, nil } -// ProgressCallback is a function type for progress updates type ProgressCallback func(current, total int) -// ItemStatusCallback is a function type for per-item status updates type ItemStatusCallback func(tweetID int64, index int, status string) // status: "success", "failed", "skipped" -// downloadTask represents a single download task type downloadTask struct { item MediaItem outputPath string index int } -// DownloadMediaWithMetadataProgressAndStatus downloads media files with progress and per-item status callbacks -// Returns: downloaded count, skipped count, failed count, error func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir string, username string, progress ProgressCallback, itemStatus ItemStatusCallback, ctx context.Context, customProxy string) (downloaded int, skipped int, failed int, err error) { if ctx == nil { ctx = context.Background() @@ -91,24 +81,20 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str return 0, 0, 0, nil } - // Prepare all tasks first (sequential to handle tweet media count) - // For bookmarks and likes, each item may have different username, so we track per username - tweetMediaCount := make(map[string]map[int64]int) // username -> tweet_id -> count + tweetMediaCount := make(map[string]map[int64]int) tasks := make([]downloadTask, 0, total) for i, item := range items { - // Use item.Username if available (for bookmarks/likes with different authors), otherwise use provided username + itemUsername := item.Username if itemUsername == "" { itemUsername = username } - // Initialize tweet media count for this username if needed if tweetMediaCount[itemUsername] == nil { tweetMediaCount[itemUsername] = make(map[int64]int) } - // Determine subfolder based on type var subfolder string switch item.Type { case "photo": @@ -123,29 +109,23 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str subfolder = "other" } - // Create base directory for this username baseDir := filepath.Join(outputDir, itemUsername) if err := os.MkdirAll(baseDir, 0755); err != nil { continue } - // Create type subfolder typeDir := filepath.Join(baseDir, subfolder) if err := os.MkdirAll(typeDir, 0755); err != nil { continue } - // Format timestamp from date timestamp := formatTimestamp(item.Date) - // Get file extension ext := getExtension(item.URL, item.Type) - // Increment counter for this username and tweet_id tweetMediaCount[itemUsername][item.TweetID]++ mediaIndex := tweetMediaCount[itemUsername][item.TweetID] - // Create filename: {username}_{timestamp}_{tweet_id}_{index}.{ext} filename := fmt.Sprintf("%s_%s_%d_%02d%s", itemUsername, timestamp, item.TweetID, mediaIndex, ext) outputPath := filepath.Join(typeDir, filename) @@ -156,27 +136,23 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str }) } - // Counters for parallel downloads var downloadedCount int64 var skippedCount int64 var failedCount int64 var completedCount int64 - // Create worker pool taskChan := make(chan downloadTask, len(tasks)) var wg sync.WaitGroup - // Start workers numWorkers := MaxConcurrentDownloads if numWorkers > len(tasks) { numWorkers = len(tasks) } - // Create HTTP client once for all workers (shared client is more efficient) var sharedClient *http.Client client, err := CreateHTTPClient(customProxy, 60*time.Second) if err != nil { - // If proxy setup fails, use default client without proxy + sharedClient = &http.Client{ Timeout: 60 * time.Second, } @@ -191,7 +167,7 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str client := sharedClient for task := range taskChan { - // Check for cancellation + select { case <-ctx.Done(): return @@ -199,17 +175,17 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str } var status string - // Skip if file already exists + if _, err := os.Stat(task.outputPath); err == nil { status = "skipped" - // Emit status immediately for skipped files + if itemStatus != nil { itemStatus(task.item.TweetID, task.index, status) } atomic.AddInt64(&skippedCount, 1) - continue // Skip to next task + continue } else if task.item.Type == "text" { - // For text tweets, write content to file + if err := os.WriteFile(task.outputPath, []byte(task.item.Content), 0644); err != nil { atomic.AddInt64(&failedCount, 1) status = "failed" @@ -221,30 +197,23 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str atomic.AddInt64(&failedCount, 1) status = "failed" } else { - // Embed metadata after successful download + tweetURL := fmt.Sprintf("https://x.com/i/status/%d", task.item.TweetID) - // Always extract original filename from URL (simpler approach) + originalFilename := ExtractOriginalFilename(task.item.URL) - - // For debugging: if original filename is still empty for video, it means it's not in the URL - // This is acceptable - video URLs from Twitter may not contain original filename - - // Embed metadata (non-fatal: if it fails, file is still downloaded) + if err := EmbedMetadata(task.outputPath, task.item.Content, tweetURL, originalFilename); err != nil { - // Log error but don't fail the download - // Metadata embedding is optional + } - + atomic.AddInt64(&downloadedCount, 1) status = "success" } - // Emit per-item status if itemStatus != nil { itemStatus(task.item.TweetID, task.index, status) } - // Update progress completed := atomic.AddInt64(&completedCount, 1) if progress != nil { progress(int(completed), total) @@ -253,7 +222,6 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str }() } - // Send tasks to workers for _, task := range tasks { select { case <-ctx.Done(): @@ -265,13 +233,11 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str } close(taskChan) - // Wait for all workers to finish wg.Wait() return int(downloadedCount), int(skippedCount), int(failedCount), nil } -// downloadFileWithContext downloads a single file with context support for cancellation func downloadFileWithContext(ctx context.Context, client *http.Client, url, outputPath string) error { req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { @@ -298,14 +264,13 @@ func downloadFileWithContext(ctx context.Context, client *http.Client, url, outp return err } -// formatTimestamp converts date string to timestamp format func formatTimestamp(dateStr string) string { - // Try parsing various date formats + formats := []string{ - "2006-01-02T15:04:05", // ISO 8601 without timezone (from extractor) - "2006-01-02T15:04:05+00:00", // ISO 8601 with timezone - "2006-01-02T15:04:05-07:00", // ISO 8601 with timezone offset - time.RFC3339, // Standard RFC3339 + "2006-01-02T15:04:05", + "2006-01-02T15:04:05+00:00", + "2006-01-02T15:04:05-07:00", + time.RFC3339, "2006-01-02T15:04:05.000Z", "2006-01-02T15:04:05Z", "2006-01-02 15:04:05", @@ -318,35 +283,30 @@ func formatTimestamp(dateStr string) string { } } - // Fallback: use empty string to indicate parsing failed return "00000000_000000" } -// getExtension determines file extension from URL and type func getExtension(mediaURL string, mediaType string) string { parsedURL, err := url.Parse(mediaURL) if err != nil { return ".jpg" } - // Check format query param for Twitter images if format := parsedURL.Query().Get("format"); format != "" { return "." + format } - // Get extension from path path := parsedURL.Path ext := filepath.Ext(path) if ext != "" { return ext } - // Default based on type switch mediaType { case "video": return ".mp4" case "gif", "animated_gif": - return ".mp4" // Twitter GIFs are actually MP4 + return ".mp4" case "text": return ".txt" default: @@ -354,28 +314,23 @@ func getExtension(mediaURL string, mediaType string) string { } } -// extractFilename extracts filename from URL (legacy) func extractFilename(mediaURL string) string { parsedURL, err := url.Parse(mediaURL) if err != nil { return fmt.Sprintf("media_%d", time.Now().UnixNano()) } - // Get the path part path := parsedURL.Path - // Extract base filename base := filepath.Base(path) - // Handle Twitter image URLs: /media/XXX -> XXX.jpg if strings.Contains(mediaURL, "pbs.twimg.com/media/") { - // Get format from query params + format := parsedURL.Query().Get("format") if format == "" { format = "jpg" } - // Remove any existing extension if idx := strings.LastIndex(base, "."); idx > 0 { base = base[:idx] } @@ -383,12 +338,10 @@ func extractFilename(mediaURL string) string { return base + "." + format } - // Handle Twitter video URLs if strings.Contains(mediaURL, "video.twimg.com") { return base } - // Default: use base name or generate one if base == "" || base == "." { return fmt.Sprintf("media_%d", time.Now().UnixNano()) } @@ -396,7 +349,6 @@ func extractFilename(mediaURL string) string { return base } -// downloadFile downloads a single file from URL func downloadFile(client *http.Client, url, outputPath string) error { resp, err := client.Get(url) if err != nil { diff --git a/backend/embed_unix.go b/backend/embed_unix.go deleted file mode 100644 index 64c34d9..0000000 --- a/backend/embed_unix.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:build linux || darwin - -package backend - -import _ "embed" - -//go:embed bin/extractor -var extractorBin []byte diff --git a/backend/embed_windows.go b/backend/embed_windows.go deleted file mode 100644 index e6121a5..0000000 --- a/backend/embed_windows.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:build windows - -package backend - -import _ "embed" - -//go:embed bin/extractor.exe -var extractorBin []byte diff --git a/backend/exiftool.go b/backend/exiftool.go index 639d6a4..823be75 100644 --- a/backend/exiftool.go +++ b/backend/exiftool.go @@ -14,26 +14,20 @@ import ( "strings" ) -// ExifTool download URLs (SourceForge) const ( - // Windows 64-bit exiftoolWindows64URL = "https://sourceforge.net/projects/exiftool/files/exiftool-13.43_64.zip/download" - // Windows 32-bit + exiftoolWindows32URL = "https://sourceforge.net/projects/exiftool/files/exiftool-13.43_32.zip/download" - // Unix (Linux/macOS): tar.gz + exiftoolUnixURL = "https://sourceforge.net/projects/exiftool/files/Image-ExifTool-13.43.tar.gz/download" ) -// ExifTool SHA256 hashes for verification -// Note: Hashes will be calculated during download if not provided -// For now, we'll skip hash verification for version 13.43 (can be added later if needed) const ( - exiftoolWindows64Hash = "" // Hash not provided, will skip verification - exiftoolWindows32Hash = "" // Hash not provided, will skip verification - exiftoolUnixHash = "" // Hash not provided, will skip verification + exiftoolWindows64Hash = "" + exiftoolWindows32Hash = "" + exiftoolUnixHash = "" ) -// GetExifToolPath returns the path to exiftool binary func GetExifToolPath() string { homeDir, _ := os.UserHomeDir() baseDir := filepath.Join(homeDir, ".twitterxmediabatchdownloader") @@ -42,27 +36,25 @@ func GetExifToolPath() string { case "windows": return filepath.Join(baseDir, "exiftool.exe") default: - // For Unix (Linux/macOS), exiftool is in Image-ExifTool-VERSION/exiftool - // We need to find the folder dynamically since version may vary + pattern := filepath.Join(baseDir, "Image-ExifTool-*") matches, err := filepath.Glob(pattern) if err == nil && len(matches) > 0 { - // Use the first match (should only be one) + exiftoolPath := filepath.Join(matches[0], "exiftool") if _, err := os.Stat(exiftoolPath); err == nil { return exiftoolPath } } - // Fallback: return path that will be created after extraction + return filepath.Join(baseDir, "exiftool") } } -// IsExifToolInstalled checks if exiftool is available func IsExifToolInstalled() bool { exiftoolPath := GetExifToolPath() if _, err := os.Stat(exiftoolPath); err == nil { - // Test if it's executable by running version command + cmd := exec.Command(exiftoolPath, "-ver") hideWindow(cmd) if err := cmd.Run(); err == nil { @@ -72,7 +64,6 @@ func IsExifToolInstalled() bool { return false } -// calculateSHA256 calculates SHA256 hash of a file func calculateSHA256(filePath string) (string, error) { file, err := os.Open(filePath) if err != nil { @@ -88,7 +79,6 @@ func calculateSHA256(filePath string) (string, error) { return hex.EncodeToString(hash.Sum(nil)), nil } -// verifyHash verifies the SHA256 hash of a downloaded file func verifyHash(filePath, expectedHash string) error { actualHash, err := calculateSHA256(filePath) if err != nil { @@ -102,12 +92,10 @@ func verifyHash(filePath, expectedHash string) error { return nil } -// is64Bit checks if the system is 64-bit func is64Bit() bool { return runtime.GOARCH == "amd64" || runtime.GOARCH == "arm64" } -// DownloadExifTool downloads exiftool binary for current platform func DownloadExifTool(progressCallback func(downloaded, total int64)) error { var downloadURL string var expectedHash string @@ -122,14 +110,13 @@ func DownloadExifTool(progressCallback func(downloaded, total int64)) error { expectedHash = exiftoolWindows32Hash } case "linux", "darwin": - // Linux and macOS use the same tar.gz archive + downloadURL = exiftoolUnixURL expectedHash = exiftoolUnixHash default: return fmt.Errorf("unsupported platform: %s", runtime.GOOS) } - // Create temp file for download tempFile, err := os.CreateTemp("", "exiftool-*") if err != nil { return fmt.Errorf("failed to create temp file: %v", err) @@ -138,7 +125,6 @@ func DownloadExifTool(progressCallback func(downloaded, total int64)) error { defer os.Remove(tempPath) defer tempFile.Close() - // Download file resp, err := http.Get(downloadURL) if err != nil { return fmt.Errorf("failed to download exiftool: %v", err) @@ -149,7 +135,6 @@ func DownloadExifTool(progressCallback func(downloaded, total int64)) error { return fmt.Errorf("failed to download exiftool: status %d", resp.StatusCode) } - // Copy with progress total := resp.ContentLength var downloaded int64 buf := make([]byte, 32*1024) @@ -175,14 +160,12 @@ func DownloadExifTool(progressCallback func(downloaded, total int64)) error { } tempFile.Close() - // Verify hash before extraction if expectedHash != "" { if err := verifyHash(tempPath, expectedHash); err != nil { return fmt.Errorf("hash verification failed: %v", err) } } - // Extract exiftool binary exiftoolPath := GetExifToolPath() baseDir := filepath.Dir(exiftoolPath) if err := os.MkdirAll(baseDir, 0755); err != nil { @@ -193,15 +176,13 @@ func DownloadExifTool(progressCallback func(downloaded, total int64)) error { case "windows": return extractExifToolFromZip(tempPath, exiftoolPath) case "linux", "darwin": - // For Linux/macOS, we need to extract and build - // For simplicity, we'll extract the exiftool script from tar.gz + return extractExifToolFromTarGz(tempPath, exiftoolPath) } return nil } -// extractExifToolFromZip extracts exiftool from Windows zip archive func extractExifToolFromZip(zipPath, destPath string) error { r, err := zip.OpenReader(zipPath) if err != nil { @@ -209,11 +190,10 @@ func extractExifToolFromZip(zipPath, destPath string) error { } defer r.Close() - // Look for exiftool(-k).exe or exiftool.exe in the zip var exiftoolFile *zip.File for _, f := range r.File { name := filepath.Base(f.Name) - // ExifTool Windows zip contains exiftool(-k).exe + if name == "exiftool(-k).exe" || name == "exiftool.exe" { exiftoolFile = f break @@ -240,12 +220,11 @@ func extractExifToolFromZip(zipPath, destPath string) error { return fmt.Errorf("failed to extract file: %v", err) } - // Also need to extract the lib directory for Windows (exiftool_files folder) libDir := filepath.Join(filepath.Dir(destPath), "exiftool_files") for _, f := range r.File { - // Check if file is in exiftool_files directory + if strings.Contains(f.Name, "exiftool_files") && !strings.HasSuffix(f.Name, "/") { - // Extract relative path from exiftool_files + parts := strings.Split(f.Name, "exiftool_files/") if len(parts) < 2 { continue @@ -257,21 +236,18 @@ func extractExifToolFromZip(zipPath, destPath string) error { targetPath := filepath.Join(libDir, relPath) - // Create directory structure if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil { continue } - // Skip if it's a directory entry if f.FileInfo().IsDir() { os.MkdirAll(targetPath, 0755) continue } - // Extract file rc, err := f.Open() if err != nil { - continue // Skip on error + continue } outFile, err := os.Create(targetPath) @@ -289,15 +265,10 @@ func extractExifToolFromZip(zipPath, destPath string) error { return nil } -// extractExifToolFromTarGz extracts exiftool from tar.gz archive (Linux/macOS) func extractExifToolFromTarGz(tarGzPath, destPath string) error { - // For Linux/macOS, ExifTool is a Perl script that requires the lib directory - // We'll extract the entire folder and use exiftool directly from Image-ExifTool-VERSION/exiftool - // This is simpler than implementing tar.gz extraction in Go baseDir := filepath.Dir(destPath) - // Use system tar command to extract cmd := exec.Command("tar", "-xzf", tarGzPath, "-C", baseDir) hideWindow(cmd) output, err := cmd.CombinedOutput() @@ -305,28 +276,23 @@ func extractExifToolFromTarGz(tarGzPath, destPath string) error { return fmt.Errorf("failed to extract tar.gz: %v, output: %s", err, string(output)) } - // Find the extracted exiftool script - // The archive extracts to Image-ExifTool-VERSION/exiftool - // We need to find the folder dynamically since version may vary var extractedDir string var exiftoolScript string - // Use glob pattern to find Image-ExifTool-* directories pattern := filepath.Join(baseDir, "Image-ExifTool-*") matches, err := filepath.Glob(pattern) if err == nil && len(matches) > 0 { - // Use the first match (should only be one) + extractedDir = matches[0] exiftoolScript = filepath.Join(extractedDir, "exiftool") } else { - // Fallback: try common version or alternative location + extractedDir = filepath.Join(baseDir, "Image-ExifTool-13.43") exiftoolScript = filepath.Join(extractedDir, "exiftool") } - // Check if exiftool script exists if _, err := os.Stat(exiftoolScript); err != nil { - // Try alternative location + extractedDir = filepath.Join(baseDir, "exiftool") exiftoolScript = filepath.Join(extractedDir, "exiftool") if _, err := os.Stat(exiftoolScript); err != nil { @@ -334,15 +300,9 @@ func extractExifToolFromTarGz(tarGzPath, destPath string) error { } } - // Make exiftool executable (required for Unix systems) if err := os.Chmod(exiftoolScript, 0755); err != nil { return fmt.Errorf("failed to make exiftool executable: %v", err) } - // For Unix, we keep the entire folder structure and use exiftool directly from it - // The folder will remain in baseDir and GetExifToolPath() will find it via glob pattern - // No need to copy or remove anything - return nil } - diff --git a/backend/extractor.go b/backend/extractor.go new file mode 100644 index 0000000..3950844 --- /dev/null +++ b/backend/extractor.go @@ -0,0 +1,369 @@ +package backend + +import ( + "archive/zip" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" +) + +const extractorReleaseAPIURL = "https://api.github.com/repos/afkarxyz/xtractor-binaries/releases/latest" + +type githubRelease struct { + TagName string `json:"tag_name"` + Assets []githubReleaseAsset `json:"assets"` +} + +type githubReleaseAsset struct { + Name string `json:"name"` + BrowserDownloadURL string `json:"browser_download_url"` + Digest string `json:"digest"` +} + +func IsExtractorInstalled() bool { + info, err := os.Stat(getExtractorPath()) + return err == nil && !info.IsDir() +} + +func DownloadExtractor(progressCallback func(downloaded, total int64)) error { + assetName, err := getExtractorAssetName() + if err != nil { + return err + } + + release, err := fetchLatestExtractorRelease() + if err != nil { + return err + } + + asset, err := findExtractorAsset(release.Assets, assetName) + if err != nil { + return err + } + + tempFile, err := os.CreateTemp("", "xtractor-*.zip") + if err != nil { + return fmt.Errorf("failed to create temp file: %v", err) + } + tempPath := tempFile.Name() + defer os.Remove(tempPath) + defer tempFile.Close() + + if err := downloadExtractorArchive(asset.BrowserDownloadURL, tempFile, progressCallback); err != nil { + return err + } + + if err := tempFile.Close(); err != nil { + return fmt.Errorf("failed to finalize xtractor archive: %v", err) + } + + if err := verifyDownloadedDigest(tempPath, asset.Digest); err != nil { + return err + } + + KillAllExtractorProcesses() + + exePath := getExtractorPath() + if err := os.MkdirAll(filepath.Dir(exePath), 0755); err != nil { + return fmt.Errorf("failed to create xtractor directory: %v", err) + } + + if err := extractExtractorArchive(tempPath, exePath); err != nil { + return err + } + + _ = removeLegacyExtractorBinary() + + return nil +} + +func requireExtractorPath() (string, error) { + exePath := getExtractorPath() + info, err := os.Stat(exePath) + if err != nil || info.IsDir() { + return "", fmt.Errorf("xtractor dependency is not installed. Please download it from the startup prompt or Settings > Core Xtractor") + } + + if runtime.GOOS != "windows" { + _ = os.Chmod(exePath, 0755) + } + + return exePath, nil +} + +func getExtractorAssetName() (string, error) { + switch runtime.GOOS { + case "windows": + switch runtime.GOARCH { + case "amd64", "arm64": + return "windows-amd64.zip", nil + } + case "linux": + if runtime.GOARCH == "amd64" { + return "linux-amd64.zip", nil + } + case "darwin": + switch resolveMacOSExtractorArch(runtime.GOARCH, detectMacOSHardwareArch()) { + case "amd64": + return "macos-amd64.zip", nil + case "arm64": + return "macos-arm64.zip", nil + } + } + + return "", fmt.Errorf("unsupported platform for prebuilt xtractor: %s/%s", runtime.GOOS, runtime.GOARCH) +} + +func resolveMacOSExtractorArch(processArch, hardwareArch string) string { + if hardwareArch == "arm64" { + return "arm64" + } + + return processArch +} + +func detectMacOSHardwareArch() string { + if runtime.GOOS != "darwin" { + return "" + } + + // On Apple Silicon, this stays "1" even if the app itself runs under Rosetta. + cmd := exec.Command("sysctl", "-in", "hw.optional.arm64") + hideWindow(cmd) + output, err := cmd.Output() + if err != nil { + return "" + } + + if strings.TrimSpace(string(output)) == "1" { + return "arm64" + } + + return "" +} + +func fetchLatestExtractorRelease() (*githubRelease, error) { + client, err := CreateHTTPClient("", 30*time.Second) + if err != nil { + return nil, fmt.Errorf("failed to configure xtractor network client: %v", err) + } + + req, err := http.NewRequest(http.MethodGet, extractorReleaseAPIURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create xtractor release request: %v", err) + } + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("User-Agent", "Twitter-X-Media-Batch-Downloader") + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch xtractor release info: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to fetch xtractor release info: status %d", resp.StatusCode) + } + + var release githubRelease + if err := json.NewDecoder(resp.Body).Decode(&release); err != nil { + return nil, fmt.Errorf("failed to decode xtractor release info: %v", err) + } + + return &release, nil +} + +func findExtractorAsset(assets []githubReleaseAsset, assetName string) (*githubReleaseAsset, error) { + for i := range assets { + if assets[i].Name == assetName { + return &assets[i], nil + } + } + + return nil, fmt.Errorf("xtractor asset %q not found in latest release", assetName) +} + +func downloadExtractorArchive(downloadURL string, destination *os.File, progressCallback func(downloaded, total int64)) error { + client, err := CreateHTTPClient("", 10*time.Minute) + if err != nil { + return fmt.Errorf("failed to configure xtractor download client: %v", err) + } + + req, err := http.NewRequest(http.MethodGet, downloadURL, nil) + if err != nil { + return fmt.Errorf("failed to create xtractor download request: %v", err) + } + req.Header.Set("User-Agent", "Twitter-X-Media-Batch-Downloader") + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("failed to download xtractor: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to download xtractor: status %d", resp.StatusCode) + } + + total := resp.ContentLength + var downloaded int64 + buf := make([]byte, 32*1024) + + for { + n, err := resp.Body.Read(buf) + if n > 0 { + if _, writeErr := destination.Write(buf[:n]); writeErr != nil { + return fmt.Errorf("failed to write xtractor archive: %v", writeErr) + } + downloaded += int64(n) + if progressCallback != nil { + progressCallback(downloaded, total) + } + } + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("failed to download xtractor: %v", err) + } + } + + return nil +} + +func verifyDownloadedDigest(filePath, digest string) error { + digest = strings.TrimSpace(digest) + if digest == "" { + return nil + } + + expectedHash := strings.TrimPrefix(strings.ToLower(digest), "sha256:") + + file, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("failed to open downloaded xtractor archive: %v", err) + } + defer file.Close() + + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return fmt.Errorf("failed to verify xtractor archive: %v", err) + } + + actualHash := hex.EncodeToString(hash.Sum(nil)) + if actualHash != expectedHash { + return fmt.Errorf("xtractor archive checksum mismatch") + } + + return nil +} + +func extractExtractorArchive(zipPath, destPath string) error { + reader, err := zip.OpenReader(zipPath) + if err != nil { + return fmt.Errorf("failed to open xtractor archive: %v", err) + } + defer reader.Close() + + entry, err := findExtractorBinaryEntry(reader.File) + if err != nil { + return err + } + + in, err := entry.Open() + if err != nil { + return fmt.Errorf("failed to open xtractor binary from archive: %v", err) + } + defer in.Close() + + tempDest := destPath + ".download" + removeTemp := true + defer func() { + if removeTemp { + _ = os.Remove(tempDest) + } + }() + + out, err := os.OpenFile(tempDest, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755) + if err != nil { + return fmt.Errorf("failed to create xtractor binary: %v", err) + } + + if _, err := io.Copy(out, in); err != nil { + out.Close() + return fmt.Errorf("failed to extract xtractor binary: %v", err) + } + + if err := out.Close(); err != nil { + return fmt.Errorf("failed to finalize xtractor binary: %v", err) + } + + if err := os.Chmod(tempDest, 0755); err != nil && runtime.GOOS != "windows" { + return fmt.Errorf("failed to make xtractor executable: %v", err) + } + + if err := os.Remove(destPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to replace existing xtractor: %v", err) + } + + if err := os.Rename(tempDest, destPath); err != nil { + return fmt.Errorf("failed to install xtractor: %v", err) + } + + removeTemp = false + return nil +} + +func findExtractorBinaryEntry(entries []*zip.File) (*zip.File, error) { + var fallback *zip.File + + for _, entry := range entries { + if entry.FileInfo().IsDir() { + continue + } + + if fallback == nil { + fallback = entry + } + + name := strings.ToLower(filepath.Base(entry.Name)) + if strings.Contains(name, "xtractor") || strings.Contains(name, "extractor") { + return entry, nil + } + } + + if fallback != nil { + return fallback, nil + } + + return nil, fmt.Errorf("xtractor binary not found in archive") +} + +func getLegacyExtractorPath() string { + homeDir, _ := os.UserHomeDir() + baseDir := filepath.Join(homeDir, ".twitterxmediabatchdownloader") + return filepath.Join(baseDir, getLegacyExecutableName()) +} + +func removeLegacyExtractorBinary() error { + oldPath := getLegacyExtractorPath() + info, err := os.Stat(oldPath) + if err != nil || info.IsDir() { + return nil + } + + if err := os.Remove(oldPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove legacy extractor binary: %v", err) + } + + return nil +} diff --git a/backend/ffmpeg.go b/backend/ffmpeg.go index ae1aa12..36f8b10 100644 --- a/backend/ffmpeg.go +++ b/backend/ffmpeg.go @@ -15,14 +15,12 @@ import ( "github.com/ulikunitz/xz" ) -// FFmpeg download URLs const ( ffmpegWindowsURL = "https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-win64-gpl.zip" ffmpegLinuxURL = "https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz" ffmpegMacOSURL = "https://evermeet.cx/ffmpeg/getrelease/ffmpeg/zip" ) -// GetFFmpegPath returns the path to ffmpeg binary func GetFFmpegPath() string { homeDir, _ := os.UserHomeDir() baseDir := filepath.Join(homeDir, ".twitterxmediabatchdownloader") @@ -35,7 +33,6 @@ func GetFFmpegPath() string { } } -// IsFFmpegInstalled checks if ffmpeg is available func IsFFmpegInstalled() bool { ffmpegPath := GetFFmpegPath() if _, err := os.Stat(ffmpegPath); err == nil { @@ -44,7 +41,6 @@ func IsFFmpegInstalled() bool { return false } -// DownloadFFmpeg downloads ffmpeg binary for current platform func DownloadFFmpeg(progressCallback func(downloaded, total int64)) error { var downloadURL string @@ -59,7 +55,6 @@ func DownloadFFmpeg(progressCallback func(downloaded, total int64)) error { return fmt.Errorf("unsupported platform: %s", runtime.GOOS) } - // Create temp file for download tempFile, err := os.CreateTemp("", "ffmpeg-*") if err != nil { return fmt.Errorf("failed to create temp file: %v", err) @@ -68,7 +63,6 @@ func DownloadFFmpeg(progressCallback func(downloaded, total int64)) error { defer os.Remove(tempPath) defer tempFile.Close() - // Download file resp, err := http.Get(downloadURL) if err != nil { return fmt.Errorf("failed to download ffmpeg: %v", err) @@ -79,7 +73,6 @@ func DownloadFFmpeg(progressCallback func(downloaded, total int64)) error { return fmt.Errorf("failed to download ffmpeg: status %d", resp.StatusCode) } - // Copy with progress total := resp.ContentLength var downloaded int64 buf := make([]byte, 32*1024) @@ -105,7 +98,6 @@ func DownloadFFmpeg(progressCallback func(downloaded, total int64)) error { } tempFile.Close() - // Extract ffmpeg binary ffmpegPath := GetFFmpegPath() baseDir := filepath.Dir(ffmpegPath) if err := os.MkdirAll(baseDir, 0755); err != nil { @@ -122,7 +114,6 @@ func DownloadFFmpeg(progressCallback func(downloaded, total int64)) error { return nil } -// extractFromZip extracts ffmpeg from zip archive func extractFromZip(zipPath, destPath string) error { r, err := zip.OpenReader(zipPath) if err != nil { @@ -131,7 +122,7 @@ func extractFromZip(zipPath, destPath string) error { defer r.Close() for _, f := range r.File { - // Look for ffmpeg binary + name := filepath.Base(f.Name) if name == "ffmpeg" || name == "ffmpeg.exe" { rc, err := f.Open() @@ -150,7 +141,6 @@ func extractFromZip(zipPath, destPath string) error { return fmt.Errorf("failed to extract file: %v", err) } - // Make executable on Unix if runtime.GOOS != "windows" { os.Chmod(destPath, 0755) } @@ -162,7 +152,6 @@ func extractFromZip(zipPath, destPath string) error { return fmt.Errorf("ffmpeg binary not found in archive") } -// extractFromTarXz extracts ffmpeg from tar.xz archive func extractFromTarXz(tarXzPath, destPath string) error { file, err := os.Open(tarXzPath) if err != nil { @@ -186,7 +175,6 @@ func extractFromTarXz(tarXzPath, destPath string) error { return fmt.Errorf("failed to read tar: %v", err) } - // Look for ffmpeg binary name := filepath.Base(header.Name) if name == "ffmpeg" && header.Typeflag == tar.TypeReg { out, err := os.Create(destPath) @@ -207,9 +195,6 @@ func extractFromTarXz(tarXzPath, destPath string) error { return fmt.Errorf("ffmpeg binary not found in archive") } -// ConvertMP4ToGIF converts an MP4 file to GIF using ffmpeg -// quality: "fast" for simple conversion, "better" for optimized palette -// resolution: "original", "high" (800px), "medium" (600px), "low" (400px) func ConvertMP4ToGIF(inputPath, outputPath, quality, resolution string) error { ffmpegPath := GetFFmpegPath() @@ -220,7 +205,7 @@ func ConvertMP4ToGIF(inputPath, outputPath, quality, resolution string) error { var args []string if quality == "fast" { - // Fast mode: simple conversion with resolution scaling + var scaleFilter string switch resolution { case "high": @@ -229,7 +214,7 @@ func ConvertMP4ToGIF(inputPath, outputPath, quality, resolution string) error { scaleFilter = "scale=600:-1" case "low": scaleFilter = "scale=400:-1" - default: // original - no scaling + default: scaleFilter = "" } @@ -250,7 +235,7 @@ func ConvertMP4ToGIF(inputPath, outputPath, quality, resolution string) error { } } } else { - // Better mode: optimized palette with dithering + var filter string switch resolution { @@ -260,11 +245,10 @@ func ConvertMP4ToGIF(inputPath, outputPath, quality, resolution string) error { filter = "scale=600:-1:flags=lanczos,palettegen=stats_mode=full[palette];[0:v]scale=600:-1:flags=lanczos[scaled];[scaled][palette]paletteuse=dither=sierra2_4a" case "low": filter = "scale=400:-1:flags=lanczos,palettegen=stats_mode=full[palette];[0:v]scale=400:-1:flags=lanczos[scaled];[scaled][palette]paletteuse=dither=sierra2_4a" - default: // original + default: filter = "palettegen=stats_mode=full[palette];[0:v][palette]paletteuse=dither=sierra2_4a" } - // Set FPS based on resolution fps := "15" if resolution == "medium" { fps = "10" @@ -282,7 +266,7 @@ func ConvertMP4ToGIF(inputPath, outputPath, quality, resolution string) error { } cmd := exec.Command(ffmpegPath, args...) - hideWindow(cmd) // Hide console window on Windows + hideWindow(cmd) output, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("ffmpeg error: %v, output: %s", err, string(output)) @@ -291,13 +275,11 @@ func ConvertMP4ToGIF(inputPath, outputPath, quality, resolution string) error { return nil } -// ConvertGIFsInFolder converts all MP4 files in gifs folder to actual GIF format func ConvertGIFsInFolder(folderPath, quality, resolution string, deleteOriginal bool) (converted int, failed int, err error) { if !IsFFmpegInstalled() { return 0, 0, fmt.Errorf("ffmpeg not installed") } - // Clean the path to handle cross-platform path separators cleanPath := filepath.Clean(folderPath) gifsFolder := filepath.Join(cleanPath, "gifs") if _, err := os.Stat(gifsFolder); os.IsNotExist(err) { diff --git a/backend/folder.go b/backend/folder.go index 29d66a8..f8608d6 100644 --- a/backend/folder.go +++ b/backend/folder.go @@ -15,9 +15,9 @@ func OpenFolderInExplorer(path string) error { switch runtime.GOOS { case "windows": - // Use cmd /c start to open explorer - more reliable + cmd = exec.Command("cmd", "/c", "start", "", path) - case "darwin": // macOS + case "darwin": cmd = exec.Command("open", path) case "linux": cmd = exec.Command("xdg-open", path) @@ -25,12 +25,12 @@ func OpenFolderInExplorer(path string) error { cmd = exec.Command("xdg-open", path) } - hideWindow(cmd) // Hide console window on Windows + hideWindow(cmd) return cmd.Run() } func SelectFolderDialog(ctx context.Context, defaultPath string) (string, error) { - // If defaultPath is empty, use default download path + if defaultPath == "" { defaultPath = GetDefaultDownloadPath() } @@ -45,7 +45,6 @@ func SelectFolderDialog(ctx context.Context, defaultPath string) (string, error) return "", err } - // If user cancelled, selectedPath will be empty if selectedPath == "" { return "", nil } @@ -53,7 +52,6 @@ func SelectFolderDialog(ctx context.Context, defaultPath string) (string, error) return selectedPath, nil } -// CheckFolderExists checks if a folder exists at the given path func CheckFolderExists(basePath, username string) bool { folderPath := filepath.Join(basePath, username) info, err := os.Stat(folderPath) @@ -63,7 +61,6 @@ func CheckFolderExists(basePath, username string) bool { return info.IsDir() } -// CheckGifsFolderExists checks if a gifs subfolder exists for the given username func CheckGifsFolderExists(basePath, username string) bool { gifsPath := filepath.Join(basePath, username, "gifs") info, err := os.Stat(gifsPath) @@ -73,7 +70,6 @@ func CheckGifsFolderExists(basePath, username string) bool { return info.IsDir() } -// CheckGifsFolderHasMP4 checks if the gifs folder has any MP4 files to convert func CheckGifsFolderHasMP4(basePath, username string) bool { gifsPath := filepath.Join(basePath, username, "gifs") @@ -93,12 +89,10 @@ func CheckGifsFolderHasMP4(basePath, username string) bool { return false } -// GetFolderPath returns the full path for a username folder func GetFolderPath(basePath, username string) string { return filepath.Join(basePath, username) } -// GetGifsFolderPath returns the full path for a username's gifs folder func GetGifsFolderPath(basePath, username string) string { return filepath.Join(basePath, username, "gifs") } diff --git a/backend/metadata.go b/backend/metadata.go index 7c42eab..bf323bd 100644 --- a/backend/metadata.go +++ b/backend/metadata.go @@ -11,11 +11,6 @@ import ( "strings" ) -// ExtractOriginalFilename extracts the original filename from Twitter media URL -// Supports 3 formats: -// 1. animated_gif: https://video.twimg.com/tweet_video/GzstJyDbIAAPuX9.mp4 -> GzstJyDbIAAPuX9 -// 2. photo: https://pbs.twimg.com/media/GynjhU0bYAAA_I4?format=jpg -> GynjhU0bYAAA_I4 -// 3. video: https://video.twimg.com/amplify_video/.../Jpo0cDa7aNtjSv7o.mp4 -> Jpo0cDa7aNtjSv7o func ExtractOriginalFilename(mediaURL string) string { parsedURL, err := url.Parse(mediaURL) if err != nil { @@ -25,16 +20,15 @@ func ExtractOriginalFilename(mediaURL string) string { path := parsedURL.Path parts := strings.Split(path, "/") - // 1. animated_gif: video.twimg.com/tweet_video/XXX.mp4 if strings.Contains(mediaURL, "/tweet_video/") { for i, part := range parts { if part == "tweet_video" && i+1 < len(parts) { filename := parts[i+1] - // Remove extension + if idx := strings.LastIndex(filename, "."); idx > 0 { filename = filename[:idx] } - // Validate alphanumeric + underscore (Twitter filenames can contain _) + if matched, _ := regexp.MatchString(`^[A-Za-z0-9_]+$`, filename); matched && len(filename) > 0 { return filename } @@ -42,32 +36,28 @@ func ExtractOriginalFilename(mediaURL string) string { } } - // 2. photo: pbs.twimg.com/media/XXX if strings.Contains(mediaURL, "pbs.twimg.com/media/") { if len(parts) > 0 { filename := parts[len(parts)-1] - // Remove extension if present + if idx := strings.LastIndex(filename, "."); idx > 0 { filename = filename[:idx] } - // Validate alphanumeric + underscore (Twitter filenames can contain _) + if matched, _ := regexp.MatchString(`^[A-Za-z0-9_]+$`, filename); matched && len(filename) > 0 { return filename } } } - // 3. video: video.twimg.com/amplify_video/.../XXX.mp4 or video.twimg.com/ext_tw_video/.../XXX.mp4 - // Filename is the last segment (before extension) - // Note: Skip if already handled by tweet_video above if strings.Contains(mediaURL, "video.twimg.com") && !strings.Contains(mediaURL, "/tweet_video/") { if len(parts) > 0 { filename := parts[len(parts)-1] - // Remove extension + if idx := strings.LastIndex(filename, "."); idx > 0 { filename = filename[:idx] } - // Validate: alphanumeric + underscore with at least one letter (to exclude pure numeric IDs) + if matched, _ := regexp.MatchString(`^[A-Za-z0-9_]+$`, filename); matched { hasLetter, _ := regexp.MatchString(`[A-Za-z]`, filename) if hasLetter && len(filename) >= 8 { @@ -80,39 +70,30 @@ func ExtractOriginalFilename(mediaURL string) string { return "" } -// EmbedMetadata embeds metadata into a media file -// Only supports JPG (images) and MP4 (videos) func EmbedMetadata(filePath string, tweetContent string, tweetURL string, originalFilename string) error { ext := strings.ToLower(filepath.Ext(filePath)) - + switch ext { case ".jpg", ".jpeg": return embedImageMetadata(filePath, tweetContent, tweetURL, originalFilename) case ".mp4": return embedVideoMetadata(filePath, tweetContent, tweetURL, originalFilename) default: - // For unsupported formats, skip metadata embedding + return nil } } -// embedImageMetadata embeds metadata into image files using exiftool or similar -// Since we don't want to add heavy dependencies, we'll use a simple approach: -// For JPEG: We can use exiftool if available, or skip if not -// For PNG: Limited support, skip for now func embedImageMetadata(filePath string, tweetContent string, tweetURL string, originalFilename string) error { - // Try to use exiftool if available (common tool for metadata) + exiftoolPath := findExifTool() if exiftoolPath == "" { - // exiftool not found, skip metadata embedding for images - // This is acceptable as it's an optional feature + return nil } - // Build metadata comment metadataComment := buildMetadataComment(tweetContent, tweetURL, originalFilename) - // Use exiftool to add comment only (URL | filename) args := []string{ "-overwrite_original", "-Comment=" + metadataComment, @@ -123,32 +104,29 @@ func embedImageMetadata(filePath string, tweetContent string, tweetURL string, o hideWindow(cmd) output, err := cmd.CombinedOutput() if err != nil { - // Non-fatal: if exiftool fails, just skip metadata + return fmt.Errorf("exiftool error (non-fatal): %v, output: %s", err, string(output)) } - _ = output // Suppress unused variable warning + _ = output return nil } -// embedVideoMetadata embeds metadata into video/GIF files using ExifTool func embedVideoMetadata(filePath string, tweetContent string, tweetURL string, originalFilename string) error { - // Use ExifTool for video metadata (works well for MP4) + exiftoolPath := findExifTool() if exiftoolPath == "" { - // ExifTool not available, skip metadata embedding (non-fatal) + return nil } return embedVideoMetadataWithExifTool(exiftoolPath, filePath, tweetContent, tweetURL, originalFilename) } -// embedVideoMetadataWithExifTool embeds metadata using ExifTool (preferred for MP4) func embedVideoMetadataWithExifTool(exiftoolPath string, filePath string, tweetContent string, tweetURL string, originalFilename string) error { - // Build metadata comment + metadataComment := buildMetadataComment(tweetContent, tweetURL, originalFilename) - // Use exiftool to add comment only (URL | filename) args := []string{ "-overwrite_original", "-Comment=" + metadataComment, @@ -159,42 +137,37 @@ func embedVideoMetadataWithExifTool(exiftoolPath string, filePath string, tweetC hideWindow(cmd) output, err := cmd.CombinedOutput() if err != nil { - // Non-fatal: if exiftool fails, skip metadata (file still downloaded) + return fmt.Errorf("exiftool error (non-fatal): %v, output: %s", err, string(output)) } - _ = output // Suppress unused variable warning + _ = output return nil } - -// buildMetadataComment builds a formatted metadata comment string func buildMetadataComment(tweetContent string, tweetURL string, originalFilename string) string { var parts []string - + if tweetURL != "" { parts = append(parts, tweetURL) } if originalFilename != "" { parts = append(parts, originalFilename) } - - // If no original filename, just return URL (don't add empty part) + if len(parts) == 0 { return "" } - + return strings.Join(parts, " | ") } -// findExifTool searches for exiftool, prioritizing the installed version in .twitterxmediabatchdownloader func findExifTool() string { - // First, check if exiftool is installed in .twitterxmediabatchdownloader + if IsExifToolInstalled() { return GetExifToolPath() } - // Fallback: check common system locations commonPaths := []string{ "exiftool", "/usr/bin/exiftool", @@ -205,23 +178,21 @@ func findExifTool() string { for _, path := range commonPaths { if runtime.GOOS == "windows" { - // On Windows, try .exe extension + if _, err := exec.LookPath(path); err == nil { return path } } else { - // On Unix, check if executable + if _, err := os.Stat(path); err == nil { return path } } } - // Try to find in PATH if path, err := exec.LookPath("exiftool"); err == nil { return path } return "" } - diff --git a/backend/twitter.go b/backend/twitter.go index 6ef995e..aa2a352 100644 --- a/backend/twitter.go +++ b/backend/twitter.go @@ -1,8 +1,6 @@ package backend import ( - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" "net/url" @@ -13,38 +11,47 @@ import ( "strings" ) -// getExecutableName returns the appropriate executable name for the current OS func getExecutableName() string { + if runtime.GOOS == "windows" { + return "xtractor.exe" + } + return "xtractor" +} + +func getLegacyExecutableName() string { if runtime.GOOS == "windows" { return "extractor.exe" } return "extractor" } -// KillAllExtractorProcesses kills all running extractor processes -// This is useful for cleanup when starting fresh or when user stops fetch func KillAllExtractorProcesses() { - exeName := getExecutableName() + names := []string{getExecutableName(), getLegacyExecutableName()} + seen := make(map[string]bool) - var cmd *exec.Cmd - if runtime.GOOS == "windows" { - // Use taskkill on Windows - cmd = exec.Command("taskkill", "/F", "/IM", exeName) - } else { - // Use pkill on Unix - cmd = exec.Command("pkill", "-f", exeName) + for _, exeName := range names { + if seen[exeName] { + continue + } + seen[exeName] = true + + var cmd *exec.Cmd + if runtime.GOOS == "windows" { + + cmd = exec.Command("taskkill", "/F", "/IM", exeName) + } else { + + cmd = exec.Command("pkill", "-f", exeName) + } + + hideWindow(cmd) + cmd.CombinedOutput() } - - hideWindow(cmd) - cmd.CombinedOutput() // Ignore errors - it's okay if no processes found } -// parseExtractorError parses the extractor output and returns a user-friendly error message -// while preserving the original error from gallery-dl func parseExtractorError(output string, username string) string { outputLower := strings.ToLower(output) - // Extract the actual error line from gallery-dl output lines := strings.Split(output, "\n") var errorLine string for _, line := range lines { @@ -58,12 +65,10 @@ func parseExtractorError(output string, username string) string { errorLine = strings.TrimSpace(output) } - // Truncate if too long if len(errorLine) > 300 { errorLine = errorLine[:300] + "..." } - // Add context hint based on error type, but keep original message var hint string if strings.Contains(outputLower, "unable to retrieve tweets from this timeline") { hint = " [Hint: End of timeline reached or rate limited - data already fetched has been saved]" @@ -80,23 +85,20 @@ func parseExtractorError(output string, username string) string { return errorLine + hint } -// TweetIDString is a custom type that unmarshals int64 but marshals as string type TweetIDString int64 -// MarshalJSON converts TweetIDString to JSON string to preserve precision in JavaScript func (t TweetIDString) MarshalJSON() ([]byte, error) { return []byte(fmt.Sprintf(`"%d"`, t)), nil } -// UnmarshalJSON accepts both number and string from JSON func (t *TweetIDString) UnmarshalJSON(data []byte) error { - // Try to unmarshal as number first (from extractor) + var num int64 if err := json.Unmarshal(data, &num); err == nil { *t = TweetIDString(num) return nil } - // Try as string (for future compatibility) + var str string if err := json.Unmarshal(data, &str); err == nil { parsed, err := fmt.Sscanf(str, "%d", &num) @@ -109,14 +111,12 @@ func (t *TweetIDString) UnmarshalJSON(data []byte) error { return fmt.Errorf("tweet_id must be number or string") } -// Author represents tweet author information from extractor type Author struct { ID int64 `json:"id"` Name string `json:"name"` Nick string `json:"nick"` } -// UserInfo represents full user information from extractor type UserInfo struct { ID int64 `json:"id"` Name string `json:"name"` @@ -137,7 +137,6 @@ type UserInfo struct { URL string `json:"url"` } -// CLIMediaItem represents a single media entry from extractor CLI type CLIMediaItem struct { URL string `json:"url"` TweetID TweetIDString `json:"tweet_id"` @@ -165,7 +164,6 @@ type CLIMediaItem struct { Sensitive bool `json:"sensitive"` } -// TweetMetadata represents tweet metadata from extractor type TweetMetadata struct { TweetID TweetIDString `json:"tweet_id"` RetweetID TweetIDString `json:"retweet_id,omitempty"` @@ -186,38 +184,35 @@ type TweetMetadata struct { Sensitive bool `json:"sensitive,omitempty"` } -// CLIResponse represents the raw response from extractor CLI type CLIResponse struct { Media []CLIMediaItem `json:"media"` Metadata []TweetMetadata `json:"metadata"` - Cursor string `json:"cursor,omitempty"` // Cursor for resume - Total int `json:"total,omitempty"` // Total items fetched - Completed bool `json:"completed,omitempty"` // True if all fetched + Cursor string `json:"cursor,omitempty"` + Total int `json:"total,omitempty"` + Completed bool `json:"completed,omitempty"` } -// TimelineEntry represents a single media entry for frontend (converted from MediaItem) type TimelineEntry struct { - URL string `json:"url"` - Date string `json:"date"` - TweetID TweetIDString `json:"tweet_id"` - Type string `json:"type"` - IsRetweet bool `json:"is_retweet"` - Extension string `json:"extension"` - Width int `json:"width"` - Height int `json:"height"` - Content string `json:"content,omitempty"` - ViewCount int `json:"view_count,omitempty"` - BookmarkCount int `json:"bookmark_count,omitempty"` - FavoriteCount int `json:"favorite_count,omitempty"` - RetweetCount int `json:"retweet_count,omitempty"` - ReplyCount int `json:"reply_count,omitempty"` - Source string `json:"source,omitempty"` - Verified bool `json:"verified,omitempty"` - OriginalFilename string `json:"original_filename,omitempty"` // Original filename from API - AuthorUsername string `json:"author_username,omitempty"` // Username of tweet author (for bookmarks and likes) + URL string `json:"url"` + Date string `json:"date"` + TweetID TweetIDString `json:"tweet_id"` + Type string `json:"type"` + IsRetweet bool `json:"is_retweet"` + Extension string `json:"extension"` + Width int `json:"width"` + Height int `json:"height"` + Content string `json:"content,omitempty"` + ViewCount int `json:"view_count,omitempty"` + BookmarkCount int `json:"bookmark_count,omitempty"` + FavoriteCount int `json:"favorite_count,omitempty"` + RetweetCount int `json:"retweet_count,omitempty"` + ReplyCount int `json:"reply_count,omitempty"` + Source string `json:"source,omitempty"` + Verified bool `json:"verified,omitempty"` + OriginalFilename string `json:"original_filename,omitempty"` + AuthorUsername string `json:"author_username,omitempty"` } -// AccountInfo represents Twitter account information (derived from metadata) type AccountInfo struct { Name string `json:"name"` Nick string `json:"nick"` @@ -228,65 +223,58 @@ type AccountInfo struct { StatusesCount int `json:"statuses_count"` } -// ExtractMetadata represents extraction metadata type ExtractMetadata struct { NewEntries int `json:"new_entries"` Page int `json:"page"` BatchSize int `json:"batch_size"` HasMore bool `json:"has_more"` - Cursor string `json:"cursor,omitempty"` // Cursor for resume capability - Completed bool `json:"completed,omitempty"` // True if all media fetched + Cursor string `json:"cursor,omitempty"` + Completed bool `json:"completed,omitempty"` } -// TwitterResponse represents the full response for frontend type TwitterResponse struct { AccountInfo AccountInfo `json:"account_info"` TotalURLs int `json:"total_urls"` Timeline []TimelineEntry `json:"timeline"` Metadata ExtractMetadata `json:"metadata"` - Cursor string `json:"cursor,omitempty"` // Cursor for next fetch - Completed bool `json:"completed,omitempty"` // True if fetch completed + Cursor string `json:"cursor,omitempty"` + Completed bool `json:"completed,omitempty"` } -// TimelineRequest represents request parameters for timeline extraction type TimelineRequest struct { Username string `json:"username"` AuthToken string `json:"auth_token"` - TimelineType string `json:"timeline_type"` // media, timeline, tweets, with_replies, likes, bookmarks - BatchSize int `json:"batch_size"` // 0 = all + TimelineType string `json:"timeline_type"` + BatchSize int `json:"batch_size"` Page int `json:"page"` - MediaType string `json:"media_type"` // all, image, video, gif + MediaType string `json:"media_type"` Retweets bool `json:"retweets"` - Cursor string `json:"cursor,omitempty"` // Resume from this cursor position + Cursor string `json:"cursor,omitempty"` } -// DateRangeRequest represents request parameters for date range extraction type DateRangeRequest struct { Username string `json:"username"` AuthToken string `json:"auth_token"` - StartDate string `json:"start_date"` // YYYY-MM-DD - EndDate string `json:"end_date"` // YYYY-MM-DD + StartDate string `json:"start_date"` + EndDate string `json:"end_date"` MediaFilter string `json:"media_filter"` Retweets bool `json:"retweets"` } -// buildTwitterURL constructs the Twitter URL based on username and timeline type func buildTwitterURL(username, timelineType string) string { - // Special case: bookmarks don't need username + if timelineType == "bookmarks" { return "https://x.com/i/bookmarks" } - // Clean username - extract handle from URL if needed username = cleanUsername(username) - // Build URL based on timeline type baseURL := "https://x.com/" + username switch timelineType { case "media": return baseURL + "/media" case "timeline": - return baseURL + "/timeline" // Best for cursor support + return baseURL + "/timeline" case "tweets": return baseURL + "/tweets" case "with_replies": @@ -294,12 +282,10 @@ func buildTwitterURL(username, timelineType string) string { case "likes": return baseURL + "/likes" default: - return baseURL + "/timeline" // Default to timeline for reliable cursor + return baseURL + "/timeline" } } -// cleanUsername extracts the handle from different input formats -// Handles: @username, username, https://x.com/username, https://x.com/username/media, etc. func cleanUsername(username string) string { username = strings.TrimSpace(username) username = strings.TrimPrefix(username, "@") @@ -311,12 +297,12 @@ func cleanUsername(username string) string { } if u, err := url.Parse(parsed); err == nil { segments := strings.Split(strings.Trim(u.Path, "/"), "/") - // Skip special paths like /i/bookmarks, /search, /home, /explore + if len(segments) > 0 && segments[0] != "" { firstSegment := strings.ToLower(segments[0]) - // These are not usernames + if firstSegment == "i" || firstSegment == "search" || firstSegment == "home" || firstSegment == "explore" || firstSegment == "settings" || firstSegment == "messages" || firstSegment == "notifications" { - return username // Return as-is, let caller handle + return username } return segments[0] } @@ -326,7 +312,6 @@ func cleanUsername(username string) string { return username } -// ensureURLScheme normalises URLs so the CLI accepts them func ensureURLScheme(raw string) string { raw = strings.TrimSpace(raw) if raw == "" { @@ -341,7 +326,6 @@ func ensureURLScheme(raw string) string { return "https://" + strings.TrimPrefix(raw, "//") } -// buildSearchURL assembles an X.com search URL for date range queries func buildSearchURL(username, startDate, endDate, mediaFilter string, includeRetweets bool) string { trimmed := strings.TrimSpace(username) lower := strings.ToLower(trimmed) @@ -380,31 +364,28 @@ func buildSearchURL(username, startDate, endDate, mediaFilter string, includeRet return fmt.Sprintf("https://x.com/search?q=%s&src=typed_query&f=live", query) } -// convertMetadataToTimelineEntry converts metadata-only tweets to timeline entries func convertMetadataToTimelineEntry(meta TweetMetadata) TimelineEntry { return TimelineEntry{ - URL: "", - Date: meta.Date, - TweetID: meta.TweetID, - Type: "text", - IsRetweet: meta.RetweetID != 0, - Extension: "txt", - Width: 0, - Height: 0, - Content: meta.Content, - ViewCount: meta.ViewCount, - BookmarkCount: meta.BookmarkCount, - FavoriteCount: meta.FavoriteCount, - RetweetCount: meta.RetweetCount, - ReplyCount: meta.ReplyCount, + URL: "", + Date: meta.Date, + TweetID: meta.TweetID, + Type: "text", + IsRetweet: meta.RetweetID != 0, + Extension: "txt", + Width: 0, + Height: 0, + Content: meta.Content, + ViewCount: meta.ViewCount, + BookmarkCount: meta.BookmarkCount, + FavoriteCount: meta.FavoriteCount, + RetweetCount: meta.RetweetCount, + ReplyCount: meta.ReplyCount, AuthorUsername: meta.Author.Name, } } -// convertToTimelineEntry converts CLIMediaItem to TimelineEntry func convertToTimelineEntry(media CLIMediaItem) TimelineEntry { - // Get username from Author field (preferred for bookmarks/likes) or User field - // Author field always contains the tweet author, while User might be the account fetching for likes + authorUsername := "" if media.Author.Name != "" { authorUsername = media.Author.Name @@ -413,26 +394,24 @@ func convertToTimelineEntry(media CLIMediaItem) TimelineEntry { } entry := TimelineEntry{ - URL: media.URL, - TweetID: media.TweetID, - Date: media.Date, - Extension: media.Extension, - Width: media.Width, - Height: media.Height, - IsRetweet: media.RetweetID != 0, - Content: media.Content, - ViewCount: media.ViewCount, - BookmarkCount: media.BookmarkCount, - FavoriteCount: media.FavoriteCount, - RetweetCount: media.RetweetCount, - ReplyCount: media.ReplyCount, - Source: media.Source, - Verified: media.Author.Verified, - AuthorUsername: authorUsername, - // OriginalFilename will be extracted from URL in download.go + URL: media.URL, + TweetID: media.TweetID, + Date: media.Date, + Extension: media.Extension, + Width: media.Width, + Height: media.Height, + IsRetweet: media.RetweetID != 0, + Content: media.Content, + ViewCount: media.ViewCount, + BookmarkCount: media.BookmarkCount, + FavoriteCount: media.FavoriteCount, + RetweetCount: media.RetweetCount, + ReplyCount: media.ReplyCount, + Source: media.Source, + Verified: media.Author.Verified, + AuthorUsername: authorUsername, } - // Determine type - media item already has type from CLI if media.Type != "" { entry.Type = media.Type } else { @@ -449,119 +428,52 @@ func convertToTimelineEntry(media CLIMediaItem) TimelineEntry { return entry } -// getExtractorPath returns the path to extractor binary -// Binary is stored in ~/.twitterxmediabatchdownloader/ (same as ffmpeg and database) func getExtractorPath() string { homeDir, _ := os.UserHomeDir() baseDir := filepath.Join(homeDir, ".twitterxmediabatchdownloader") return filepath.Join(baseDir, getExecutableName()) } -// getHashFilePath returns the path to the hash file for version checking -func getHashFilePath() string { - homeDir, _ := os.UserHomeDir() - baseDir := filepath.Join(homeDir, ".twitterxmediabatchdownloader") - return filepath.Join(baseDir, "extractor.sha256") -} - -// calculateHash calculates SHA256 hash of data -func calculateHash(data []byte) string { - hash := sha256.Sum256(data) - return hex.EncodeToString(hash[:]) -} - -// ensureExtractor ensures the extractor binary exists -// Extracts from embedded binary if not present or if hash differs (update) -func ensureExtractor() (string, error) { - exePath := getExtractorPath() - hashPath := getHashFilePath() - baseDir := filepath.Dir(exePath) - - // Create directory if not exists - if err := os.MkdirAll(baseDir, 0755); err != nil { - return "", fmt.Errorf("failed to create directory: %v", err) - } - - // Calculate hash of embedded binary - embeddedHash := calculateHash(extractorBin) - - // Check if binary and hash file exist - if _, err := os.Stat(exePath); err == nil { - // Binary exists - check hash - if storedHash, err := os.ReadFile(hashPath); err == nil { - if string(storedHash) == embeddedHash { - return exePath, nil // Already extracted and up to date - } - } - // Hash differs or missing - need to update - os.Remove(exePath) - } - - // Extract binary - if err := os.WriteFile(exePath, extractorBin, 0755); err != nil { - return "", fmt.Errorf("failed to write extractor: %v", err) - } - - // Save hash for future comparison - if err := os.WriteFile(hashPath, []byte(embeddedHash), 0644); err != nil { - // Non-fatal - binary still works, just won't skip next time - fmt.Printf("Warning: failed to save hash file: %v\n", err) - } - - return exePath, nil -} - -// ExtractTimeline extracts media from user timeline using the new CLI func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) { - // Get or extract extractor binary (persistent, not temp) - exePath, err := ensureExtractor() + + exePath, err := requireExtractorPath() if err != nil { return nil, err } - // Determine the right endpoint based on what user wants: - // - Media (all/image/video/gif): Use /media endpoint - fastest and most reliable - // - Text tweets: Use /tweets endpoint with --text-tweets - // - With retweets: Use /tweets endpoint (retweets not available on /media) isTextOnly := req.MediaType == "text" wantsRetweets := req.Retweets timelineType := req.TimelineType if timelineType == "" { if isTextOnly { - // Text-only tweets need /tweets endpoint + timelineType = "tweets" } else if wantsRetweets { - // Retweets need /tweets endpoint (not available on /media) + timelineType = "tweets" } else { - // Default: /media endpoint - fastest for media-only fetch + timelineType = "media" } } url := buildTwitterURL(req.Username, timelineType) - // Build command arguments for new CLI format - // Format: extractor.exe URL --auth-token TOKEN --json [options] args := []string{url} - // Add auth token if req.AuthToken != "" { args = append(args, "--auth-token", req.AuthToken) } else { args = append(args, "--guest") } - // Always request JSON output with metadata args = append(args, "--json", "--metadata") - // Add limit if specified if req.BatchSize > 0 { args = append(args, "--limit", fmt.Sprintf("%d", req.BatchSize)) } - // Handle retweets - only relevant when using /tweets endpoint if timelineType == "tweets" || timelineType == "timeline" { if req.Retweets { args = append(args, "--retweets", "include") @@ -570,12 +482,10 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) { } } - // Only add --text-tweets when explicitly requesting text content if isTextOnly { args = append(args, "--text-tweets") } - // Handle media type filter using --type parameter if req.MediaType != "" && req.MediaType != "all" && !isTextOnly { switch req.MediaType { case "image": @@ -587,21 +497,18 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) { } } - // Add cursor for resume capability if req.Cursor != "" { args = append(args, "--cursor", req.Cursor) } - // Execute command with UTF-8 encoding cmd := exec.Command(exePath, args...) cmd.Env = append(os.Environ(), "PYTHONIOENCODING=utf-8", "PYTHONUTF8=1", ) - hideWindow(cmd) // Hide console window on Windows + hideWindow(cmd) output, err := cmd.CombinedOutput() - // Ensure process is killed after completion if cmd.Process != nil { cmd.Process.Kill() } @@ -612,36 +519,31 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) { return nil, fmt.Errorf("%s", errorMsg) } - // Find JSON in output (skip any info messages) jsonStr := extractJSON(string(output)) if jsonStr == "" { outputStr := string(output) if strings.TrimSpace(outputStr) == "" { - return nil, fmt.Errorf("empty_response: Extractor returned no data. The timeline may be empty or inaccessible") + return nil, fmt.Errorf("empty_response: Xtractor returned no data. The timeline may be empty or inaccessible") } - return nil, fmt.Errorf("parse_error: Could not parse extractor output. Raw output: %s", outputStr) + return nil, fmt.Errorf("parse_error: Could not parse xtractor output. Raw output: %s", outputStr) } - // Parse CLI response var cliResponse CLIResponse if err := json.Unmarshal([]byte(jsonStr), &cliResponse); err != nil { return nil, fmt.Errorf("json_error: Failed to parse JSON response: %v", err) } - // Convert to frontend format var timeline []TimelineEntry accountInfo := AccountInfo{ Name: req.Username, Nick: req.Username, } - // Build a set of tweet IDs that have media mediaTweetIDs := make(map[int64]bool) for _, media := range cliResponse.Media { mediaTweetIDs[int64(media.TweetID)] = true } - // For bookmarks and likes, keep name as "bookmarks"/"likes" (not from author tweet) isBookmarks := req.TimelineType == "bookmarks" isLikes := req.TimelineType == "likes" if isBookmarks { @@ -653,7 +555,7 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) { } if isTextOnly { - // Text-only mode: get tweets from metadata that don't have media + timeline = make([]TimelineEntry, 0) for _, meta := range cliResponse.Metadata { if !mediaTweetIDs[int64(meta.TweetID)] { @@ -661,7 +563,6 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) { } } - // Get account info from first media item if available, otherwise from metadata if !isBookmarks && !isLikes { if len(cliResponse.Media) > 0 { user := cliResponse.Media[0].User @@ -678,7 +579,7 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) { accountInfo.Nick = firstMeta.Author.Nick } } else { - // For bookmarks and likes, get other info from first media item if available + if len(cliResponse.Media) > 0 { user := cliResponse.Media[0].User accountInfo.Date = user.Date @@ -689,15 +590,13 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) { } } } else if len(cliResponse.Media) > 0 { - // Normal media items only (text tweets handled separately with "text" media type) + timeline = make([]TimelineEntry, 0, len(cliResponse.Media)) - // Add media items for _, media := range cliResponse.Media { timeline = append(timeline, convertToTimelineEntry(media)) } - // Get account info from first media item user := cliResponse.Media[0].User if !isBookmarks && !isLikes { accountInfo.Name = user.Name @@ -709,29 +608,29 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) { accountInfo.ProfileImage = user.ProfileImage accountInfo.StatusesCount = user.StatusesCount } else if len(cliResponse.Metadata) > 0 { - // Fallback: Text-only tweets (no media) - convert metadata to timeline entries + timeline = make([]TimelineEntry, 0, len(cliResponse.Metadata)) for _, meta := range cliResponse.Metadata { entry := TimelineEntry{ - URL: "", // No media URL for text tweets - TweetID: meta.TweetID, - Date: meta.Date, - Type: "text", - IsRetweet: meta.RetweetID != 0, - Extension: "txt", - Width: 0, - Height: 0, - Content: meta.Content, - ViewCount: meta.ViewCount, - BookmarkCount: meta.BookmarkCount, - FavoriteCount: meta.FavoriteCount, - RetweetCount: meta.RetweetCount, - ReplyCount: meta.ReplyCount, + URL: "", + TweetID: meta.TweetID, + Date: meta.Date, + Type: "text", + IsRetweet: meta.RetweetID != 0, + Extension: "txt", + Width: 0, + Height: 0, + Content: meta.Content, + ViewCount: meta.ViewCount, + BookmarkCount: meta.BookmarkCount, + FavoriteCount: meta.FavoriteCount, + RetweetCount: meta.RetweetCount, + ReplyCount: meta.ReplyCount, AuthorUsername: meta.Author.Name, } timeline = append(timeline, entry) } - // Get account info from first metadata + if !isBookmarks && !isLikes { firstMeta := cliResponse.Metadata[0] accountInfo.Name = firstMeta.Author.Name @@ -739,7 +638,6 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) { } } - // Determine if there's more data to fetch hasMore := cliResponse.Cursor != "" && !cliResponse.Completed response := &TwitterResponse{ @@ -761,10 +659,9 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) { return response, nil } -// ExtractDateRange extracts media based on date range using the new CLI func ExtractDateRange(req DateRangeRequest) (*TwitterResponse, error) { - // Get or extract extractor binary (persistent, not temp) - exePath, err := ensureExtractor() + + exePath, err := requireExtractorPath() if err != nil { return nil, err } @@ -772,17 +669,14 @@ func ExtractDateRange(req DateRangeRequest) (*TwitterResponse, error) { mediaFilter := strings.ToLower(strings.TrimSpace(req.MediaFilter)) url := buildSearchURL(req.Username, req.StartDate, req.EndDate, mediaFilter, req.Retweets) - // Build command arguments args := []string{url} - // Add auth token if req.AuthToken != "" { args = append(args, "--auth-token", req.AuthToken) } else { args = append(args, "--guest") } - // Always request JSON output with metadata args = append(args, "--json", "--metadata") if req.Retweets { @@ -796,7 +690,6 @@ func ExtractDateRange(req DateRangeRequest) (*TwitterResponse, error) { args = append(args, "--text-tweets") } - // Execute command with UTF-8 encoding cmd := exec.Command(exePath, args...) cmd.Env = append(os.Environ(), "PYTHONIOENCODING=utf-8", @@ -805,7 +698,6 @@ func ExtractDateRange(req DateRangeRequest) (*TwitterResponse, error) { hideWindow(cmd) output, err := cmd.CombinedOutput() - // Ensure process is killed after completion if cmd.Process != nil { cmd.Process.Kill() } @@ -816,23 +708,20 @@ func ExtractDateRange(req DateRangeRequest) (*TwitterResponse, error) { return nil, fmt.Errorf("%s", errorMsg) } - // Find JSON in output (skip any info messages) jsonStr := extractJSON(string(output)) if jsonStr == "" { outputStr := string(output) if strings.TrimSpace(outputStr) == "" { - return nil, fmt.Errorf("empty_response: Extractor returned no data. The timeline may be empty or inaccessible") + return nil, fmt.Errorf("empty_response: Xtractor returned no data. The timeline may be empty or inaccessible") } - return nil, fmt.Errorf("parse_error: Could not parse extractor output. Raw output: %s", outputStr) + return nil, fmt.Errorf("parse_error: Could not parse xtractor output. Raw output: %s", outputStr) } - // Parse CLI response var cliResponse CLIResponse if err := json.Unmarshal([]byte(jsonStr), &cliResponse); err != nil { return nil, fmt.Errorf("json_error: Failed to parse JSON response: %v", err) } - // Convert to frontend format mediaTweetIDs := make(map[int64]bool) for _, media := range cliResponse.Media { mediaTweetIDs[int64(media.TweetID)] = true @@ -851,7 +740,6 @@ func ExtractDateRange(req DateRangeRequest) (*TwitterResponse, error) { } } - // Build account info from first media item (has full user info) accountInfo := AccountInfo{ Name: req.Username, Nick: req.Username, @@ -871,7 +759,6 @@ func ExtractDateRange(req DateRangeRequest) (*TwitterResponse, error) { accountInfo.Nick = firstMeta.Author.Nick } - // Determine if there's more data to fetch hasMore := cliResponse.Cursor != "" && !cliResponse.Completed response := &TwitterResponse{ @@ -893,15 +780,13 @@ func ExtractDateRange(req DateRangeRequest) (*TwitterResponse, error) { return response, nil } -// extractJSON finds and extracts JSON object from output string func extractJSON(output string) string { - // Find the start of JSON object + start := strings.Index(output, "{") if start == -1 { return "" } - // Find the matching closing brace depth := 0 for i := start; i < len(output); i++ { if output[i] == '{' { diff --git a/frontend/eslint.config.js b/frontend/eslint.config.js index 5e6b472..13d2e6b 100644 --- a/frontend/eslint.config.js +++ b/frontend/eslint.config.js @@ -1,23 +1,22 @@ -import js from '@eslint/js' -import globals from 'globals' -import reactHooks from 'eslint-plugin-react-hooks' -import reactRefresh from 'eslint-plugin-react-refresh' -import tseslint from 'typescript-eslint' -import { defineConfig, globalIgnores } from 'eslint/config' - +import js from '@eslint/js'; +import globals from 'globals'; +import reactHooks from 'eslint-plugin-react-hooks'; +import reactRefresh from 'eslint-plugin-react-refresh'; +import tseslint from 'typescript-eslint'; +import { defineConfig, globalIgnores } from 'eslint/config'; export default defineConfig([ - globalIgnores(['dist']), - { - files: ['**/*.{ts,tsx}'], - extends: [ - js.configs.recommended, - tseslint.configs.recommended, - reactHooks.configs.flat.recommended, - reactRefresh.configs.vite, - ], - languageOptions: { - ecmaVersion: 2020, - globals: globals.browser, + globalIgnores(['dist']), + { + files: ['**/*.{ts,tsx}'], + extends: [ + js.configs.recommended, + tseslint.configs.recommended, + reactHooks.configs.flat.recommended, + reactRefresh.configs.vite, + ], + languageOptions: { + ecmaVersion: 2020, + globals: globals.browser, + }, }, - }, -]) +]); diff --git a/frontend/package.json.md5 b/frontend/package.json.md5 index ac6d48d..2d07d73 100644 --- a/frontend/package.json.md5 +++ b/frontend/package.json.md5 @@ -1 +1 @@ -93300717a79529efe3943af0fda63408 \ No newline at end of file +cb4545ab700ecc9d641ab1be76b622f1 \ No newline at end of file diff --git a/frontend/scripts/generate-icon.js b/frontend/scripts/generate-icon.js index b975dd8..02eec68 100644 --- a/frontend/scripts/generate-icon.js +++ b/frontend/scripts/generate-icon.js @@ -2,32 +2,23 @@ import sharp from 'sharp'; import { readFileSync, mkdirSync } from 'fs'; import { dirname, join } from 'path'; import { fileURLToPath } from 'url'; - const __dirname = dirname(fileURLToPath(import.meta.url)); const rootDir = join(__dirname, '..', '..'); - const svgPath = join(rootDir, 'frontend', 'public', 'icon.svg'); const outputPath = join(rootDir, 'build', 'appicon.png'); - async function generateIcon() { - try { - // Ensure build directory exists - mkdirSync(join(rootDir, 'build'), { recursive: true }); - - // Read SVG - const svgBuffer = readFileSync(svgPath); - - // Convert SVG to PNG (1024x1024 for Wails) - await sharp(svgBuffer) - .resize(1024, 1024) - .png() - .toFile(outputPath); - - console.log('✓ Icon generated:', outputPath); - } catch (error) { - console.error('✗ Failed to generate icon:', error.message); - process.exit(1); - } + try { + mkdirSync(join(rootDir, 'build'), { recursive: true }); + const svgBuffer = readFileSync(svgPath); + await sharp(svgBuffer) + .resize(1024, 1024) + .png() + .toFile(outputPath); + console.log('✓ Icon generated:', outputPath); + } + catch (error) { + console.error('✗ Failed to generate icon:', error.message); + process.exit(1); + } } - generateIcon(); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 78fcdb0..0ce4e9e 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -4,20 +4,7 @@ import { getSettings, applyThemeMode, applyFont } from "@/lib/settings"; import { applyTheme } from "@/lib/themes"; import { toastWithSound as toast } from "@/lib/toast-with-sound"; import { logger } from "@/lib/logger"; -import { - saveFetchState, - getFetchState, - clearFetchState, - getResumableInfo, - stateToResponse, - mergeTimelines, - saveCursor, - getCursor, - clearCursor, - type FetchState, -} from "@/lib/fetch-state"; - -// Components +import { saveFetchState, getFetchState, clearFetchState, getResumableInfo, stateToResponse, mergeTimelines, saveCursor, getCursor, clearCursor, type FetchState, } from "@/lib/fetch-state"; import { TitleBar } from "@/components/TitleBar"; import { Sidebar, type PageType } from "@/components/Sidebar"; import { Header } from "@/components/Header"; @@ -26,1803 +13,1299 @@ import { MediaList } from "@/components/MediaList"; import { DatabaseView } from "@/components/DatabaseView"; import { SettingsPage } from "@/components/SettingsPage"; import { DebugLoggerPage } from "@/components/DebugLoggerPage"; +import { DependencySetupDialog } from "@/components/DependencySetupDialog"; import type { HistoryItem } from "@/components/FetchHistory"; import type { TwitterResponse } from "@/types/api"; - -// Wails bindings import { ExtractTimeline, ExtractDateRange, SaveAccountToDBWithStatus, CleanupExtractorProcesses, GetAllAccountsFromDB } from "../wailsjs/go/main/App"; - const HISTORY_KEY = "twitter_media_fetch_history"; const MAX_HISTORY = 10; -const CURRENT_VERSION = "4.3"; -const BATCH_SIZE = 200; // Fetch in batches for progressive display and resume - +const CURRENT_VERSION = __APP_VERSION__; +const BATCH_SIZE = 200; function formatNumberWithComma(num: number): string { - return num.toLocaleString(); + return num.toLocaleString(); } - function App() { - const [currentPage, setCurrentPage] = useState("main"); - const [username, setUsername] = useState(""); - const [loading, setLoading] = useState(false); - const [result, setResult] = useState(null); - const [fetchedMediaType, setFetchedMediaType] = useState("all"); - const [hasUpdate, setHasUpdate] = useState(false); - const [releaseDate, setReleaseDate] = useState(null); - const [fetchHistory, setFetchHistory] = useState([]); - const [resumeInfo, setResumeInfo] = useState<{ canResume: boolean; mediaCount: number } | null>(null); - const [elapsedTime, setElapsedTime] = useState(0); - const [remainingTime, setRemainingTime] = useState(null); - const [newMediaCount, setNewMediaCount] = useState(null); - const stopFetchRef = useRef(false); - const fetchStartTimeRef = useRef(null); - const timeoutIntervalRef = useRef(null); - - // Multiple mode state - const [fetchType, setFetchType] = useState("single"); - const [multipleAccounts, setMultipleAccounts] = useState([]); - const [isFetchingAll, setIsFetchingAll] = useState(false); - - // Mode state for SearchBar - const [searchMode, setSearchMode] = useState("public"); - const [searchPrivateType, setSearchPrivateType] = useState("bookmarks"); - const stopAllRef = useRef(false); - const accountTimersRef = useRef>(new Map()); - const accountStartTimesRef = useRef>(new Map()); - const accountStopFlagsRef = useRef>(new Map()); - const accountMediaCountRef = useRef>(new Map()); - const accountTimeoutSecondsRef = useRef>(new Map()); - - // Handle fetch type change - const handleFetchTypeChange = (type: FetchType) => { - setFetchType(type); - if (type === "single") { - // Clear multiple accounts when switching to single - setMultipleAccounts([]); - setIsFetchingAll(false); - stopAllRef.current = true; - // Clear all timers - accountTimersRef.current.forEach((interval) => clearInterval(interval)); - accountTimersRef.current.clear(); - accountStartTimesRef.current.clear(); - accountTimeoutSecondsRef.current.clear(); - accountStopFlagsRef.current.clear(); - accountMediaCountRef.current.clear(); - } else { - // Clear result when switching to multiple - setResult(null); - } - }; - - useEffect(() => { - const settings = getSettings(); - applyThemeMode(settings.themeMode); - applyTheme(settings.theme); - applyFont(settings.fontFamily); - - const mediaQuery = window.matchMedia("(prefers-color-scheme: dark)"); - const handleChange = () => { - const currentSettings = getSettings(); - if (currentSettings.themeMode === "auto") { - applyThemeMode("auto"); - applyTheme(currentSettings.theme); - } + const [currentPage, setCurrentPage] = useState("main"); + const [username, setUsername] = useState(""); + const [loading, setLoading] = useState(false); + const [result, setResult] = useState(null); + const [fetchedMediaType, setFetchedMediaType] = useState("all"); + const [hasUpdate, setHasUpdate] = useState(false); + const [releaseDate, setReleaseDate] = useState(null); + const [fetchHistory, setFetchHistory] = useState([]); + const [resumeInfo, setResumeInfo] = useState<{ + canResume: boolean; + mediaCount: number; + } | null>(null); + const [elapsedTime, setElapsedTime] = useState(0); + const [remainingTime, setRemainingTime] = useState(null); + const [newMediaCount, setNewMediaCount] = useState(null); + const stopFetchRef = useRef(false); + const fetchStartTimeRef = useRef(null); + const timeoutIntervalRef = useRef(null); + const [fetchType, setFetchType] = useState("single"); + const [multipleAccounts, setMultipleAccounts] = useState([]); + const [isFetchingAll, setIsFetchingAll] = useState(false); + const [searchMode, setSearchMode] = useState("public"); + const [searchPrivateType, setSearchPrivateType] = useState("bookmarks"); + const stopAllRef = useRef(false); + const accountTimersRef = useRef>(new Map()); + const accountStartTimesRef = useRef>(new Map()); + const accountStopFlagsRef = useRef>(new Map()); + const accountMediaCountRef = useRef>(new Map()); + const accountTimeoutSecondsRef = useRef>(new Map()); + const dbSaveWarningShownRef = useRef(false); + const reportDatabaseSaveError = (scope: string, error: unknown) => { + console.error(`Failed to save ${scope} to database:`, error); + if (!dbSaveWarningShownRef.current) { + dbSaveWarningShownRef.current = true; + toast.warning("Fetch completed, but saving to Saved Accounts failed"); + } }; - - mediaQuery.addEventListener("change", handleChange); - checkForUpdates(); - loadHistory(); - - return () => { - mediaQuery.removeEventListener("change", handleChange); + const handleFetchTypeChange = (type: FetchType) => { + setFetchType(type); + if (type === "single") { + setMultipleAccounts([]); + setIsFetchingAll(false); + stopAllRef.current = true; + accountTimersRef.current.forEach((interval) => clearInterval(interval)); + accountTimersRef.current.clear(); + accountStartTimesRef.current.clear(); + accountTimeoutSecondsRef.current.clear(); + accountStopFlagsRef.current.clear(); + accountMediaCountRef.current.clear(); + } + else { + setResult(null); + } }; - }, []); - - const checkForUpdates = async () => { - try { - const response = await fetch( - "https://api.github.com/repos/afkarxyz/Twitter-X-Media-Batch-Downloader/releases/latest" - ); - const data = await response.json(); - const latestVersion = data.tag_name?.replace(/^v/, "") || ""; - - if (data.published_at) { - setReleaseDate(data.published_at); - } - - if (latestVersion && latestVersion > CURRENT_VERSION) { - setHasUpdate(true); - } - } catch (err) { - console.error("Failed to check for updates:", err); - } - }; - - const loadHistory = () => { - try { - const saved = localStorage.getItem(HISTORY_KEY); - if (saved) { - setFetchHistory(JSON.parse(saved)); - } - } catch (err) { - console.error("Failed to load history:", err); - } - }; - - const saveHistory = (history: HistoryItem[]) => { - try { - localStorage.setItem(HISTORY_KEY, JSON.stringify(history)); - } catch (err) { - console.error("Failed to save history:", err); - } - }; - - const addToHistory = (data: TwitterResponse, inputUsername: string) => { - // Clean username (remove @ and extract from URL if needed) - let cleanUsername = inputUsername.trim(); - if (cleanUsername.startsWith("@")) { - cleanUsername = cleanUsername.slice(1); - } - if (cleanUsername.includes("x.com/") || cleanUsername.includes("twitter.com/")) { - const match = cleanUsername.match(/(?:x\.com|twitter\.com)\/([^/?]+)/); - if (match) cleanUsername = match[1]; - } - - setFetchHistory((prev) => { - // Use username from API response (account_info.name) for consistency - const apiUsername = data.account_info.name; - const filtered = prev.filter((h) => h.username.toLowerCase() !== apiUsername.toLowerCase()); - const newItem: HistoryItem = { - id: crypto.randomUUID(), - username: apiUsername, // username/handle from API - name: data.account_info.nick, // display name from API - image: data.account_info.profile_image, - mediaCount: data.total_urls, - timestamp: Date.now(), - }; - const updated = [newItem, ...filtered].slice(0, MAX_HISTORY); - saveHistory(updated); - return updated; - }); - }; - - const removeFromHistory = (id: string) => { - setFetchHistory((prev) => { - const updated = prev.filter((h) => h.id !== id); - saveHistory(updated); - return updated; - }); - }; - - const handleHistorySelect = (item: HistoryItem) => { - setUsername(item.username); - }; - - // Check for resumable fetch when username changes - const checkResumable = useCallback((user: string) => { - if (!user.trim()) { - setResumeInfo(null); - return; - } - const info = getResumableInfo(user.trim()); - setResumeInfo(info.canResume ? { canResume: true, mediaCount: info.mediaCount } : null); - }, []); - - useEffect(() => { - checkResumable(username); - }, [username, checkResumable]); - - const handleStopFetch = useCallback(async () => { - stopFetchRef.current = true; - logger.info("Stopping fetch..."); - toast.info("Stopping..."); - // Kill any running extractor processes - try { - await CleanupExtractorProcesses(); - } catch { - // Ignore cleanup errors - } - }, []); - - // Timer effect for fetch timeout - useEffect(() => { - if (loading && fetchStartTimeRef.current !== null) { - const settings = getSettings(); - const timeoutSeconds = settings.fetchTimeout || 60; - - // Update timer every second - timeoutIntervalRef.current = window.setInterval(() => { - if (fetchStartTimeRef.current !== null) { - const elapsed = Math.floor((Date.now() - fetchStartTimeRef.current) / 1000); - setElapsedTime(elapsed); - const remaining = Math.max(0, timeoutSeconds - elapsed); - setRemainingTime(remaining); - - // Auto-stop if timeout reached - if (remaining <= 0) { - stopFetchRef.current = true; - logger.warning("Fetch timeout reached. Stopping..."); - toast.warning("Fetch timeout reached. Stopping..."); - handleStopFetch(); - } - } - }, 1000); - } else { - if (timeoutIntervalRef.current !== null) { - clearInterval(timeoutIntervalRef.current); - timeoutIntervalRef.current = null; - } - setElapsedTime(0); - setRemainingTime(null); - fetchStartTimeRef.current = null; - } - - return () => { - if (timeoutIntervalRef.current !== null) { - clearInterval(timeoutIntervalRef.current); - timeoutIntervalRef.current = null; - } - }; - }, [loading, handleStopFetch]); - - const handleFetch = async ( - useDateRange: boolean, - startDate?: string, - endDate?: string, - mediaType?: string, - retweets?: boolean, - mode: FetchMode = "public", - privateType?: PrivateType, - authToken?: string, - isResume?: boolean - ) => { - // Prevent multiple concurrent fetches - if (loading) { - logger.warning("Fetch already in progress, please wait or stop first"); - toast.warning("Fetch already in progress"); - return; - } - - // Validate based on mode - const isBookmarks = mode === "private" && privateType === "bookmarks"; - const isLikes = mode === "private" && privateType === "likes"; - - // Username required for public mode and likes mode - if (!isBookmarks && !username.trim()) { - toast.error("Please enter a username"); - return; - } - - // Use auth token from SearchBar - if (!authToken?.trim()) { - toast.error("Please enter your auth token"); - return; - } - - // Reset state for new fetch - setLoading(true); - setFetchedMediaType(mediaType || "all"); - stopFetchRef.current = false; - fetchStartTimeRef.current = Date.now(); - setElapsedTime(0); - setNewMediaCount(null); - - // Get timeout from settings - const settings = getSettings(); - const timeoutSeconds = settings.fetchTimeout || 60; - setRemainingTime(timeoutSeconds); - - // Cleanup any leftover processes from previous fetch - try { - await CleanupExtractorProcesses(); - } catch { - // Ignore cleanup errors - } - - // Determine what we're fetching for logging - const fetchTarget = isBookmarks - ? "your bookmarks" - : isLikes - ? "your likes" - : `@${username}`; - - // Check for resume state - const cleanUsername = isBookmarks ? "bookmarks" : isLikes ? "likes" : username.trim(); - let existingState: FetchState | null = null; - let cursor: string | undefined; - let allTimeline: TwitterResponse["timeline"] = []; - let accountInfo: TwitterResponse["account_info"] | null = null; - - if (isResume) { - existingState = getFetchState(cleanUsername); - if (existingState && existingState.cursor && !existingState.completed) { - cursor = existingState.cursor; - allTimeline = existingState.timeline; - accountInfo = existingState.accountInfo; - logger.info(`Resuming ${fetchTarget} from ${allTimeline.length} items...`); - - // Show existing data immediately - if (accountInfo) { - setResult({ - account_info: accountInfo, - timeline: allTimeline, - total_urls: allTimeline.length, - metadata: { new_entries: 0, page: 0, batch_size: 0, has_more: true }, - }); - } - } else { - toast.error("No resumable fetch found"); - setLoading(false); - return; - } - } else { - // Fresh fetch - clear any existing state - clearFetchState(cleanUsername); - setResult(null); - logger.info(`Fetching ${fetchTarget}...`); - } - - try { - let finalData: TwitterResponse | null = null; - - if (useDateRange && startDate && endDate && mode === "public") { - // Date range mode - single fetch (only for public mode) - logger.info(`Using date range: ${startDate} to ${endDate}`); - - // Check timeout before date range fetch - if (fetchStartTimeRef.current !== null) { - const settings = getSettings(); - const timeoutSeconds = settings.fetchTimeout || 60; - const elapsed = Math.floor((Date.now() - fetchStartTimeRef.current) / 1000); - if (elapsed >= timeoutSeconds) { - logger.warning(`Timeout reached (${timeoutSeconds}s). Stopping fetch...`); - stopFetchRef.current = true; - toast.warning("Fetch timeout reached. Stopping..."); - setLoading(false); - return; - } - } - - const response = await ExtractDateRange({ - username: username.trim(), - auth_token: authToken.trim(), - start_date: startDate, - end_date: endDate, - media_filter: mediaType || "all", - retweets: retweets || false, - }); - finalData = JSON.parse(response); - - // Save to database immediately for date range mode - if (finalData && finalData.account_info) { - try { - await SaveAccountToDBWithStatus( - finalData.account_info.name, - finalData.account_info.nick, - finalData.account_info.profile_image, - finalData.total_urls, - JSON.stringify(finalData), - mediaType || "all", - finalData.cursor || "", - finalData.completed ?? true - ); - } catch (err) { - console.error("Failed to save date range data to database:", err); - } - } - - // Save to database immediately for date range mode - if (finalData && finalData.account_info) { - try { - await SaveAccountToDBWithStatus( - finalData.account_info.name, - finalData.account_info.nick, - finalData.account_info.profile_image, - finalData.total_urls, - JSON.stringify(finalData), - mediaType || "all", - finalData.cursor || "", - finalData.completed ?? true - ); - } catch (err) { - console.error("Failed to save date range data to database:", err); - } - } - } else { - // Timeline mode - check fetch mode from settings + useEffect(() => { const settings = getSettings(); - const isSingleMode = settings.fetchMode === "single"; - const batchSize = isSingleMode ? 0 : BATCH_SIZE; - - let timelineType = "timeline"; - if (isBookmarks) { - timelineType = "bookmarks"; - } else if (isLikes) { - timelineType = "likes"; + applyThemeMode(settings.themeMode); + applyTheme(settings.theme); + applyFont(settings.fontFamily); + const mediaQuery = window.matchMedia("(prefers-color-scheme: dark)"); + const handleChange = () => { + const currentSettings = getSettings(); + if (currentSettings.themeMode === "auto") { + applyThemeMode("auto"); + applyTheme(currentSettings.theme); + } + }; + mediaQuery.addEventListener("change", handleChange); + checkForUpdates(); + loadHistory(); + return () => { + mediaQuery.removeEventListener("change", handleChange); + }; + }, []); + const checkForUpdates = async () => { + try { + const response = await fetch("https://api.github.com/repos/afkarxyz/Twitter-X-Media-Batch-Downloader/releases/latest"); + const data = await response.json(); + const latestVersion = data.tag_name?.replace(/^v/, "") || ""; + if (data.published_at) { + setReleaseDate(data.published_at); + } + if (latestVersion && latestVersion > CURRENT_VERSION) { + setHasUpdate(true); + } } - - let hasMore = true; - let page = 0; - - // Single mode: one fetch, no loop - // Batch mode: loop until done or stopped - while (hasMore && !stopFetchRef.current) { - // Check timeout before each batch - if (fetchStartTimeRef.current !== null) { + catch (err) { + console.error("Failed to check for updates:", err); + } + }; + const loadHistory = () => { + try { + const saved = localStorage.getItem(HISTORY_KEY); + if (saved) { + setFetchHistory(JSON.parse(saved)); + } + } + catch (err) { + console.error("Failed to load history:", err); + } + }; + const saveHistory = (history: HistoryItem[]) => { + try { + localStorage.setItem(HISTORY_KEY, JSON.stringify(history)); + } + catch (err) { + console.error("Failed to save history:", err); + } + }; + const addToHistory = (data: TwitterResponse, inputUsername: string) => { + let cleanUsername = inputUsername.trim(); + if (cleanUsername.startsWith("@")) { + cleanUsername = cleanUsername.slice(1); + } + if (cleanUsername.includes("x.com/") || cleanUsername.includes("twitter.com/")) { + const match = cleanUsername.match(/(?:x\.com|twitter\.com)\/([^/?]+)/); + if (match) + cleanUsername = match[1]; + } + setFetchHistory((prev) => { + const apiUsername = data.account_info.name; + const filtered = prev.filter((h) => h.username.toLowerCase() !== apiUsername.toLowerCase()); + const newItem: HistoryItem = { + id: crypto.randomUUID(), + username: apiUsername, + name: data.account_info.nick, + image: data.account_info.profile_image, + mediaCount: data.total_urls, + timestamp: Date.now(), + }; + const updated = [newItem, ...filtered].slice(0, MAX_HISTORY); + saveHistory(updated); + return updated; + }); + }; + const removeFromHistory = (id: string) => { + setFetchHistory((prev) => { + const updated = prev.filter((h) => h.id !== id); + saveHistory(updated); + return updated; + }); + }; + const handleHistorySelect = (item: HistoryItem) => { + setUsername(item.username); + }; + const checkResumable = useCallback((user: string) => { + if (!user.trim()) { + setResumeInfo(null); + return; + } + const info = getResumableInfo(user.trim()); + setResumeInfo(info.canResume ? { canResume: true, mediaCount: info.mediaCount } : null); + }, []); + useEffect(() => { + checkResumable(username); + }, [username, checkResumable]); + const handleStopFetch = useCallback(async () => { + stopFetchRef.current = true; + logger.info("Stopping fetch..."); + toast.info("Stopping..."); + try { + await CleanupExtractorProcesses(); + } + catch { + } + }, []); + useEffect(() => { + if (loading && fetchStartTimeRef.current !== null) { const settings = getSettings(); const timeoutSeconds = settings.fetchTimeout || 60; - const elapsed = Math.floor((Date.now() - fetchStartTimeRef.current) / 1000); - if (elapsed >= timeoutSeconds) { - logger.warning(`Timeout reached (${timeoutSeconds}s). Stopping fetch...`); - stopFetchRef.current = true; - toast.warning("Fetch timeout reached. Stopping..."); - - // IMPORTANT: Save state immediately before breaking out of loop - // This ensures no data is lost when timeout occurs - if (accountInfo && allTimeline.length > 0) { - saveFetchState({ - username: cleanUsername, - cursor: cursor || "", - timeline: allTimeline, - accountInfo: accountInfo, - totalFetched: allTimeline.length, - completed: false, - authToken: authToken.trim(), - mediaType: mediaType || "all", - retweets: retweets || false, - timelineType: timelineType, - }); - - // Also save to database immediately - const timeoutResponse: TwitterResponse = { - account_info: accountInfo, - timeline: allTimeline, - total_urls: allTimeline.length, - metadata: { - new_entries: 0, - page: page, - batch_size: batchSize, - has_more: hasMore, - cursor: cursor, - completed: false, - }, - cursor: cursor, - completed: false, - }; - try { - await SaveAccountToDBWithStatus( - accountInfo.name, - accountInfo.nick, - accountInfo.profile_image, - allTimeline.length, - JSON.stringify(timeoutResponse), - mediaType || "all", - cursor || "", - false - ); - logger.info(`Saved ${allTimeline.length} items before timeout`); - } catch (err) { - console.error("Failed to save timeout state to database:", err); + timeoutIntervalRef.current = window.setInterval(() => { + if (fetchStartTimeRef.current !== null) { + const elapsed = Math.floor((Date.now() - fetchStartTimeRef.current) / 1000); + setElapsedTime(elapsed); + const remaining = Math.max(0, timeoutSeconds - elapsed); + setRemainingTime(remaining); + if (remaining <= 0) { + stopFetchRef.current = true; + logger.warning("Fetch timeout reached. Stopping..."); + toast.warning("Fetch timeout reached. Stopping..."); + handleStopFetch(); + } } - } - break; + }, 1000); + } + else { + if (timeoutIntervalRef.current !== null) { + clearInterval(timeoutIntervalRef.current); + timeoutIntervalRef.current = null; } - } - - const batchNum = page + 1; - logger.info(isSingleMode ? "Fetching all media..." : `Fetching batch ${batchNum}${cursor ? " (resuming)" : ""}...`); - - const response = await ExtractTimeline({ - username: isBookmarks ? "" : username.trim(), - auth_token: authToken.trim(), - timeline_type: timelineType, - batch_size: batchSize, - page: page, - media_type: mediaType || "all", - retweets: retweets || false, - cursor: cursor, - }); - - const data: TwitterResponse = JSON.parse(response); - - // Set account info from first response - if (!accountInfo && data.account_info) { - accountInfo = data.account_info; - if (isBookmarks) { - // For bookmarks, ensure name is "bookmarks" and nick is "My Bookmarks" - accountInfo.name = "bookmarks"; - accountInfo.nick = "My Bookmarks"; - } else if (isLikes) { - // For likes, ensure name is "likes" and nick is "My Likes" - accountInfo.name = "likes"; - accountInfo.nick = "My Likes"; + setElapsedTime(0); + setRemainingTime(null); + fetchStartTimeRef.current = null; + } + return () => { + if (timeoutIntervalRef.current !== null) { + clearInterval(timeoutIntervalRef.current); + timeoutIntervalRef.current = null; } - } - - // Merge new entries (deduplicate) - const previousCount = allTimeline.length; - allTimeline = mergeTimelines(allTimeline, data.timeline); - const newCount = allTimeline.length - previousCount; - - // Update new media count for display - if (newCount > 0) { - setNewMediaCount(newCount); - // Clear after 1 second - setTimeout(() => setNewMediaCount(null), 1000); - } - - // Update cursor for next batch - cursor = data.cursor; - hasMore = !!data.cursor && !data.completed; - page++; - - // Save cursor every batch (lightweight, just a string) - if (cursor) { - saveCursor(cleanUsername, cursor); - } - - // Update UI progressively - show results immediately - if (accountInfo) { - setResult({ - account_info: accountInfo, - timeline: allTimeline, - total_urls: allTimeline.length, - metadata: { - new_entries: data.timeline.length, - page: page, - batch_size: batchSize, - has_more: hasMore, - cursor: cursor, - completed: !hasMore, - }, - cursor: cursor, - completed: !hasMore, - }); - } - - logger.info(`Fetched ${allTimeline.length} items total`); - - // Save state EVERY batch to ensure no data loss on timeout/error - // This is critical for large accounts where fetch can take a long time - saveFetchState({ - username: cleanUsername, - cursor: cursor || "", - timeline: allTimeline, - accountInfo: accountInfo, - totalFetched: allTimeline.length, - completed: !hasMore, - authToken: authToken.trim(), - mediaType: mediaType || "all", - retweets: retweets || false, - timelineType: timelineType, - }); - - // Save to database immediately after each batch for realtime persistence - // This ensures data is saved even if fetch fails mid-way - if (accountInfo) { - const currentResponse: TwitterResponse = { - account_info: accountInfo, - timeline: allTimeline, - total_urls: allTimeline.length, - metadata: { - new_entries: data.timeline.length, - page: page, - batch_size: batchSize, - has_more: hasMore, - cursor: cursor, - completed: !hasMore, - }, - cursor: cursor, - completed: !hasMore, + }; + }, [loading, handleStopFetch]); + const handleFetch = async (useDateRange: boolean, startDate?: string, endDate?: string, mediaType?: string, retweets?: boolean, mode: FetchMode = "public", privateType?: PrivateType, authToken?: string, isResume?: boolean) => { + if (loading) { + logger.warning("Fetch already in progress, please wait or stop first"); + toast.warning("Fetch already in progress"); + return; + } + const isBookmarks = mode === "private" && privateType === "bookmarks"; + const isLikes = mode === "private" && privateType === "likes"; + if (!isBookmarks && !username.trim()) { + toast.error("Please enter a username"); + return; + } + if (!authToken?.trim()) { + toast.error("Please enter your auth token"); + return; + } + setLoading(true); + setFetchedMediaType(mediaType || "all"); + stopFetchRef.current = false; + fetchStartTimeRef.current = Date.now(); + setElapsedTime(0); + setNewMediaCount(null); + const settings = getSettings(); + const timeoutSeconds = settings.fetchTimeout || 60; + setRemainingTime(timeoutSeconds); + try { + await CleanupExtractorProcesses(); + } + catch { + } + const fetchTarget = isBookmarks + ? "your bookmarks" + : isLikes + ? "your likes" + : `@${username}`; + const cleanUsername = isBookmarks ? "bookmarks" : isLikes ? "likes" : username.trim(); + let existingState: FetchState | null = null; + let cursor: string | undefined; + let allTimeline: TwitterResponse["timeline"] = []; + let accountInfo: TwitterResponse["account_info"] | null = null; + if (isResume) { + existingState = getFetchState(cleanUsername); + if (existingState && existingState.cursor && !existingState.completed) { + cursor = existingState.cursor; + allTimeline = existingState.timeline; + accountInfo = existingState.accountInfo; + logger.info(`Resuming ${fetchTarget} from ${allTimeline.length} items...`); + if (accountInfo) { + setResult({ + account_info: accountInfo, + timeline: allTimeline, + total_urls: allTimeline.length, + metadata: { new_entries: 0, page: 0, batch_size: 0, has_more: true }, + }); + } + } + else { + toast.error("No resumable fetch found"); + setLoading(false); + return; + } + } + else { + clearFetchState(cleanUsername); + setResult(null); + logger.info(`Fetching ${fetchTarget}...`); + } + try { + let finalData: TwitterResponse | null = null; + if (useDateRange && startDate && endDate && mode === "public") { + logger.info(`Using date range: ${startDate} to ${endDate}`); + if (fetchStartTimeRef.current !== null) { + const settings = getSettings(); + const timeoutSeconds = settings.fetchTimeout || 60; + const elapsed = Math.floor((Date.now() - fetchStartTimeRef.current) / 1000); + if (elapsed >= timeoutSeconds) { + logger.warning(`Timeout reached (${timeoutSeconds}s). Stopping fetch...`); + stopFetchRef.current = true; + toast.warning("Fetch timeout reached. Stopping..."); + setLoading(false); + return; + } + } + const response = await ExtractDateRange({ + username: username.trim(), + auth_token: authToken.trim(), + start_date: startDate, + end_date: endDate, + media_filter: mediaType || "all", + retweets: retweets || false, + }); + finalData = JSON.parse(response); + if (finalData && finalData.account_info) { + try { + await SaveAccountToDBWithStatus(finalData.account_info.name, finalData.account_info.nick, finalData.account_info.profile_image, finalData.total_urls, JSON.stringify(finalData), mediaType || "all", finalData.cursor || "", finalData.completed ?? true); + } + catch (err) { + reportDatabaseSaveError("date range data", err); + } + } + if (finalData && finalData.account_info) { + try { + await SaveAccountToDBWithStatus(finalData.account_info.name, finalData.account_info.nick, finalData.account_info.profile_image, finalData.total_urls, JSON.stringify(finalData), mediaType || "all", finalData.cursor || "", finalData.completed ?? true); + } + catch (err) { + reportDatabaseSaveError("date range data", err); + } + } + } + else { + const settings = getSettings(); + const isSingleMode = settings.fetchMode === "single"; + const batchSize = isSingleMode ? 0 : BATCH_SIZE; + let timelineType = "timeline"; + if (isBookmarks) { + timelineType = "bookmarks"; + } + else if (isLikes) { + timelineType = "likes"; + } + let hasMore = true; + let page = 0; + while (hasMore && !stopFetchRef.current) { + if (fetchStartTimeRef.current !== null) { + const settings = getSettings(); + const timeoutSeconds = settings.fetchTimeout || 60; + const elapsed = Math.floor((Date.now() - fetchStartTimeRef.current) / 1000); + if (elapsed >= timeoutSeconds) { + logger.warning(`Timeout reached (${timeoutSeconds}s). Stopping fetch...`); + stopFetchRef.current = true; + toast.warning("Fetch timeout reached. Stopping..."); + if (accountInfo && allTimeline.length > 0) { + saveFetchState({ + username: cleanUsername, + cursor: cursor || "", + timeline: allTimeline, + accountInfo: accountInfo, + totalFetched: allTimeline.length, + completed: false, + authToken: authToken.trim(), + mediaType: mediaType || "all", + retweets: retweets || false, + timelineType: timelineType, + }); + const timeoutResponse: TwitterResponse = { + account_info: accountInfo, + timeline: allTimeline, + total_urls: allTimeline.length, + metadata: { + new_entries: 0, + page: page, + batch_size: batchSize, + has_more: hasMore, + cursor: cursor, + completed: false, + }, + cursor: cursor, + completed: false, + }; + try { + await SaveAccountToDBWithStatus(accountInfo.name, accountInfo.nick, accountInfo.profile_image, allTimeline.length, JSON.stringify(timeoutResponse), mediaType || "all", cursor || "", false); + logger.info(`Saved ${allTimeline.length} items before timeout`); + } + catch (err) { + reportDatabaseSaveError("timeout state", err); + } + } + break; + } + } + const batchNum = page + 1; + logger.info(isSingleMode ? "Fetching all media..." : `Fetching batch ${batchNum}${cursor ? " (resuming)" : ""}...`); + const response = await ExtractTimeline({ + username: isBookmarks ? "" : username.trim(), + auth_token: authToken.trim(), + timeline_type: timelineType, + batch_size: batchSize, + page: page, + media_type: mediaType || "all", + retweets: retweets || false, + cursor: cursor, + }); + const data: TwitterResponse = JSON.parse(response); + if (!accountInfo && data.account_info) { + accountInfo = data.account_info; + if (isBookmarks) { + accountInfo.name = "bookmarks"; + accountInfo.nick = "My Bookmarks"; + } + else if (isLikes) { + accountInfo.name = "likes"; + accountInfo.nick = "My Likes"; + } + } + const previousCount = allTimeline.length; + allTimeline = mergeTimelines(allTimeline, data.timeline); + const newCount = allTimeline.length - previousCount; + if (newCount > 0) { + setNewMediaCount(newCount); + setTimeout(() => setNewMediaCount(null), 1000); + } + cursor = data.cursor; + hasMore = !!data.cursor && !data.completed; + page++; + if (cursor) { + saveCursor(cleanUsername, cursor); + } + if (accountInfo) { + setResult({ + account_info: accountInfo, + timeline: allTimeline, + total_urls: allTimeline.length, + metadata: { + new_entries: data.timeline.length, + page: page, + batch_size: batchSize, + has_more: hasMore, + cursor: cursor, + completed: !hasMore, + }, + cursor: cursor, + completed: !hasMore, + }); + } + logger.info(`Fetched ${allTimeline.length} items total`); + saveFetchState({ + username: cleanUsername, + cursor: cursor || "", + timeline: allTimeline, + accountInfo: accountInfo, + totalFetched: allTimeline.length, + completed: !hasMore, + authToken: authToken.trim(), + mediaType: mediaType || "all", + retweets: retweets || false, + timelineType: timelineType, + }); + if (accountInfo) { + const currentResponse: TwitterResponse = { + account_info: accountInfo, + timeline: allTimeline, + total_urls: allTimeline.length, + metadata: { + new_entries: data.timeline.length, + page: page, + batch_size: batchSize, + has_more: hasMore, + cursor: cursor, + completed: !hasMore, + }, + cursor: cursor, + completed: !hasMore, + }; + try { + await SaveAccountToDBWithStatus(accountInfo.name, accountInfo.nick, accountInfo.profile_image, allTimeline.length, JSON.stringify(currentResponse), mediaType || "all", cursor || "", !hasMore); + } + catch (err) { + reportDatabaseSaveError("fetch progress", err); + } + } + } + if (stopFetchRef.current) { + const elapsedSecs = fetchStartTimeRef.current ? Math.floor((Date.now() - fetchStartTimeRef.current) / 1000) : 0; + logger.info(`Stopped at ${allTimeline.length} items - can resume later (${elapsedSecs}s)`); + toast.info(`Stopped at ${formatNumberWithComma(allTimeline.length)} items`); + setResumeInfo({ canResume: true, mediaCount: allTimeline.length }); + } + else { + clearFetchState(cleanUsername); + clearCursor(cleanUsername); + setResumeInfo(null); + } + finalData = accountInfo + ? { + account_info: accountInfo, + timeline: allTimeline, + total_urls: allTimeline.length, + metadata: { + new_entries: allTimeline.length, + page: page, + batch_size: batchSize, + has_more: false, + cursor: cursor, + completed: !stopFetchRef.current, + }, + cursor: cursor, + completed: !stopFetchRef.current, + } + : null; + } + if (finalData) { + setResult(finalData); + if (mode === "public") { + addToHistory(finalData, username); + } + if (!useDateRange) { + try { + await SaveAccountToDBWithStatus(finalData.account_info.name, finalData.account_info.nick, finalData.account_info.profile_image, finalData.total_urls, JSON.stringify(finalData), mediaType || "all", finalData.cursor || "", finalData.completed ?? true); + } + catch (err) { + reportDatabaseSaveError("final fetch status", err); + } + } + if (!stopFetchRef.current) { + const elapsedSecs = fetchStartTimeRef.current ? Math.floor((Date.now() - fetchStartTimeRef.current) / 1000) : 0; + logger.success(`Found ${finalData.total_urls} media items (${elapsedSecs}s)`); + toast.success(`${finalData.total_urls} media items found`); + } + } + } + catch (error) { + const errorMsg = error instanceof Error ? error.message : String(error); + const elapsedSecs = fetchStartTimeRef.current ? Math.floor((Date.now() - fetchStartTimeRef.current) / 1000) : 0; + logger.error(`Failed to fetch: ${errorMsg} (${elapsedSecs}s)`); + if (errorMsg.toLowerCase().includes("xtractor dependency") || errorMsg.toLowerCase().includes("extractor dependency")) { + toast.error(errorMsg); + } + else { + toast.error("Failed to fetch media"); + } + const savedState = getFetchState(cleanUsername); + if (savedState && savedState.timeline.length > 0) { + const partialResponse = stateToResponse(savedState); + if (partialResponse) { + setResult(partialResponse); + setResumeInfo({ canResume: true, mediaCount: savedState.timeline.length }); + toast.info(`Saved ${formatNumberWithComma(savedState.timeline.length)} items - can resume`); + try { + await SaveAccountToDBWithStatus(partialResponse.account_info.name, partialResponse.account_info.nick, partialResponse.account_info.profile_image, partialResponse.total_urls, JSON.stringify(partialResponse), mediaType || "all", savedState.cursor || "", false); + } + catch (dbErr) { + reportDatabaseSaveError("partial fetch data", dbErr); + } + } + } + } + finally { + setLoading(false); + fetchStartTimeRef.current = null; + setElapsedTime(0); + setRemainingTime(null); + } + }; + const handleResume = (authToken: string, mediaType?: string, retweets?: boolean) => { + handleFetch(false, undefined, undefined, mediaType, retweets, "public", undefined, authToken, true); + }; + const handleClearResume = () => { + if (username.trim()) { + clearFetchState(username.trim()); + setResumeInfo(null); + toast.info("Resume data cleared"); + } + }; + const handleLoadFromDB = (responseJSON: string, loadedUsername: string) => { + try { + const data: TwitterResponse = JSON.parse(responseJSON); + setResult(data); + setUsername(loadedUsername); + setCurrentPage("main"); + setFetchType("single"); + const isPrivate = loadedUsername === "bookmarks" || loadedUsername === "likes"; + if (isPrivate) { + if (loadedUsername === "bookmarks") { + setSearchMode("private"); + setSearchPrivateType("bookmarks"); + } + else if (loadedUsername === "likes") { + setSearchMode("private"); + setSearchPrivateType("likes"); + } + } + else { + setSearchMode("public"); + } + toast.success(`Loaded @${loadedUsername} from database`); + } + catch (error) { + toast.error("Failed to parse saved data"); + } + }; + const parseUsername = (input: string): string => { + let clean = input.trim(); + if (clean.startsWith("@")) { + clean = clean.slice(1); + } + if (clean.includes("x.com/") || clean.includes("twitter.com/")) { + const match = clean.match(/(?:x\.com|twitter\.com)\/([^/?]+)/); + if (match) + clean = match[1]; + } + return clean; + }; + const handleUpdateSelected = (usernames: string[]) => { + if (usernames.length === 0) { + toast.error("No accounts selected"); + return; + } + const accounts: MultipleAccount[] = usernames.map((username) => { + const cleanUsername = parseUsername(username); + return { + id: crypto.randomUUID(), + username: cleanUsername, + status: "pending", + mediaCount: 0, + previousMediaCount: 0, + elapsedTime: 0, + remainingTime: null, + showDiff: false, }; + }); + setMultipleAccounts((prev) => { + const existingUsernames = new Set(prev.map((acc) => acc.username.toLowerCase())); + const newAccounts = accounts.filter((acc) => !existingUsernames.has(acc.username.toLowerCase())); + return [...prev, ...newAccounts]; + }); + setFetchType("multiple"); + setCurrentPage("main"); + toast.success(`Added ${formatNumberWithComma(accounts.length)} account(s) to multiple fetch`); + }; + const handleImportFile = () => { + const input = document.createElement("input"); + input.type = "file"; + input.accept = ".txt"; + input.onchange = async (e) => { + const file = (e.target as HTMLInputElement).files?.[0]; + if (!file) + return; try { - await SaveAccountToDBWithStatus( - accountInfo.name, - accountInfo.nick, - accountInfo.profile_image, - allTimeline.length, - JSON.stringify(currentResponse), - mediaType || "all", - cursor || "", - !hasMore - ); - } catch (err) { - console.error("Failed to save progress to database:", err); + const text = await file.text(); + const lines = text.split("\n").map((line) => line.trim()).filter((line) => line.length > 0); + if (lines.length === 0) { + toast.error("File is empty"); + return; + } + const accounts: MultipleAccount[] = lines.map((line) => { + const username = parseUsername(line); + return { + id: crypto.randomUUID(), + username, + status: "pending", + mediaCount: 0, + previousMediaCount: 0, + elapsedTime: 0, + remainingTime: null, + showDiff: false, + }; + }); + setMultipleAccounts(accounts); + toast.success(`Imported ${formatNumberWithComma(accounts.length)} account(s)`); } - } - } - - // If stopped by user, keep state for resume - if (stopFetchRef.current) { - const elapsedSecs = fetchStartTimeRef.current ? Math.floor((Date.now() - fetchStartTimeRef.current) / 1000) : 0; - logger.info(`Stopped at ${allTimeline.length} items - can resume later (${elapsedSecs}s)`); - toast.info(`Stopped at ${formatNumberWithComma(allTimeline.length)} items`); - setResumeInfo({ canResume: true, mediaCount: allTimeline.length }); - } else { - // Completed - clear state - clearFetchState(cleanUsername); - clearCursor(cleanUsername); - setResumeInfo(null); - } - - // Build final response - finalData = accountInfo - ? { - account_info: accountInfo, - timeline: allTimeline, - total_urls: allTimeline.length, - metadata: { - new_entries: allTimeline.length, - page: page, - batch_size: batchSize, - has_more: false, - cursor: cursor, - completed: !stopFetchRef.current, - }, - cursor: cursor, - completed: !stopFetchRef.current, + catch (error) { + toast.error("Failed to read file"); } - : null; - } - - if (finalData) { - setResult(finalData); - - // Only add to history for public mode - if (mode === "public") { - addToHistory(finalData, username); + }; + input.click(); + }; + const handleFetchAll = async () => { + if (multipleAccounts.length === 0) { + toast.error("No accounts to fetch"); + return; } - - // Note: Data is already saved to database after each batch for timeline mode - // and immediately after fetch for date range mode, so we don't need to save again here - // unless it's the final completion status update for timeline mode - if (!useDateRange) { - // For timeline mode, final save is already done in the loop after each batch - // This is just for ensuring final status is correct - try { - await SaveAccountToDBWithStatus( - finalData.account_info.name, - finalData.account_info.nick, - finalData.account_info.profile_image, - finalData.total_urls, - JSON.stringify(finalData), - mediaType || "all", - finalData.cursor || "", - finalData.completed ?? true - ); - } catch (err) { - console.error("Failed to save final status to database:", err); - } + setIsFetchingAll(true); + stopAllRef.current = false; + const PUBLIC_AUTH_TOKEN_KEY = "twitter_public_auth_token"; + const authToken = localStorage.getItem(PUBLIC_AUTH_TOKEN_KEY) || ""; + if (!authToken.trim()) { + toast.error("Please enter your auth token"); + setIsFetchingAll(false); + return; } - - if (!stopFetchRef.current) { - const elapsedSecs = fetchStartTimeRef.current ? Math.floor((Date.now() - fetchStartTimeRef.current) / 1000) : 0; - logger.success(`Found ${finalData.total_urls} media items (${elapsedSecs}s)`); - toast.success(`${finalData.total_urls} media items found`); - } - } - } catch (error) { - const errorMsg = error instanceof Error ? error.message : String(error); - const elapsedSecs = fetchStartTimeRef.current ? Math.floor((Date.now() - fetchStartTimeRef.current) / 1000) : 0; - logger.error(`Failed to fetch: ${errorMsg} (${elapsedSecs}s)`); - toast.error("Failed to fetch media"); - - // On error, check if we have partial data saved - const savedState = getFetchState(cleanUsername); - if (savedState && savedState.timeline.length > 0) { - const partialResponse = stateToResponse(savedState); - if (partialResponse) { - setResult(partialResponse); - setResumeInfo({ canResume: true, mediaCount: savedState.timeline.length }); - toast.info(`Saved ${formatNumberWithComma(savedState.timeline.length)} items - can resume`); - - // Also save partial data to database for persistence - try { - await SaveAccountToDBWithStatus( - partialResponse.account_info.name, - partialResponse.account_info.nick, - partialResponse.account_info.profile_image, - partialResponse.total_urls, - JSON.stringify(partialResponse), - mediaType || "all", - savedState.cursor || "", - false // not completed - ); - } catch (dbErr) { - console.error("Failed to save partial data to database:", dbErr); - } - } - } - } finally { - setLoading(false); - // Reset timer when fetch completes or stops - fetchStartTimeRef.current = null; - setElapsedTime(0); - setRemainingTime(null); - } - }; - - // Handle resume fetch - const handleResume = (authToken: string, mediaType?: string, retweets?: boolean) => { - handleFetch(false, undefined, undefined, mediaType, retweets, "public", undefined, authToken, true); - }; - - // Clear resume state - const handleClearResume = () => { - if (username.trim()) { - clearFetchState(username.trim()); - setResumeInfo(null); - toast.info("Resume data cleared"); - } - }; - - const handleLoadFromDB = (responseJSON: string, loadedUsername: string) => { - try { - const data: TwitterResponse = JSON.parse(responseJSON); - setResult(data); - setUsername(loadedUsername); - setCurrentPage("main"); - - // Set fetch type to single when loading from database - setFetchType("single"); - - // Determine if account is private (bookmarks or likes) - const isPrivate = loadedUsername === "bookmarks" || loadedUsername === "likes"; - - // Set mode based on account type - if (isPrivate) { - if (loadedUsername === "bookmarks") { - setSearchMode("private"); - setSearchPrivateType("bookmarks"); - } else if (loadedUsername === "likes") { - setSearchMode("private"); - setSearchPrivateType("likes"); - } - } else { - setSearchMode("public"); - } - - toast.success(`Loaded @${loadedUsername} from database`); - } catch (error) { - toast.error("Failed to parse saved data"); - } - }; - - // Parse username from various formats - const parseUsername = (input: string): string => { - let clean = input.trim(); - if (clean.startsWith("@")) { - clean = clean.slice(1); - } - if (clean.includes("x.com/") || clean.includes("twitter.com/")) { - const match = clean.match(/(?:x\.com|twitter\.com)\/([^/?]+)/); - if (match) clean = match[1]; - } - return clean; - }; - - // Handle update selected accounts from database view - const handleUpdateSelected = (usernames: string[]) => { - if (usernames.length === 0) { - toast.error("No accounts selected"); - return; - } - - // Parse usernames (same as handleImportFile) - const accounts: MultipleAccount[] = usernames.map((username) => { - const cleanUsername = parseUsername(username); - return { - id: crypto.randomUUID(), - username: cleanUsername, - status: "pending", - mediaCount: 0, - previousMediaCount: 0, - elapsedTime: 0, - remainingTime: null, - showDiff: false, - }; - }); - - // Merge with existing accounts (avoid duplicates by username) - setMultipleAccounts((prev) => { - const existingUsernames = new Set(prev.map((acc) => acc.username.toLowerCase())); - const newAccounts = accounts.filter( - (acc) => !existingUsernames.has(acc.username.toLowerCase()) - ); - return [...prev, ...newAccounts]; - }); - - // Set fetch type to multiple and navigate to home - setFetchType("multiple"); - setCurrentPage("main"); - toast.success(`Added ${formatNumberWithComma(accounts.length)} account(s) to multiple fetch`); - }; - - // Handle import txt file - const handleImportFile = () => { - const input = document.createElement("input"); - input.type = "file"; - input.accept = ".txt"; - input.onchange = async (e) => { - const file = (e.target as HTMLInputElement).files?.[0]; - if (!file) return; - - try { - const text = await file.text(); - const lines = text.split("\n").map((line) => line.trim()).filter((line) => line.length > 0); - - if (lines.length === 0) { - toast.error("File is empty"); - return; - } - - const accounts: MultipleAccount[] = lines.map((line) => { - const username = parseUsername(line); - return { - id: crypto.randomUUID(), - username, - status: "pending", + const settings = getSettings(); + const timeoutSeconds = settings.fetchTimeout || 60; + setMultipleAccounts((prev) => prev.map((acc) => ({ + ...acc, + status: "pending" as const, mediaCount: 0, previousMediaCount: 0, elapsedTime: 0, - remainingTime: null, + remainingTime: timeoutSeconds, + error: undefined, showDiff: false, - }; - }); - - setMultipleAccounts(accounts); - toast.success(`Imported ${formatNumberWithComma(accounts.length)} account(s)`); - } catch (error) { - toast.error("Failed to read file"); - } + }))); + let currentAccounts = [...multipleAccounts]; + for (let i = 0; i < currentAccounts.length; i++) { + if (stopAllRef.current) { + break; + } + const account = currentAccounts[i]; + const accountId = account.id; + accountStopFlagsRef.current.set(accountId, false); + setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId + ? { ...acc, status: "fetching" as const } + : acc)); + const settings = getSettings(); + const timeoutSeconds = settings.fetchTimeout || 60; + accountStartTimesRef.current.set(accountId, Date.now()); + accountTimeoutSecondsRef.current.set(accountId, timeoutSeconds); + setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId + ? { ...acc, elapsedTime: 0, remainingTime: timeoutSeconds } + : acc)); + const timerInterval = window.setInterval(() => { + const startTime = accountStartTimesRef.current.get(accountId); + const timeoutSecs = accountTimeoutSecondsRef.current.get(accountId) || timeoutSeconds; + if (startTime) { + const elapsed = Math.floor((Date.now() - startTime) / 1000); + const remaining = Math.max(0, timeoutSecs - elapsed); + setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId + ? { ...acc, elapsedTime: elapsed, remainingTime: remaining } + : acc)); + if (remaining <= 0) { + accountStopFlagsRef.current.set(accountId, true); + clearInterval(timerInterval); + accountTimersRef.current.delete(accountId); + accountStartTimesRef.current.delete(accountId); + accountTimeoutSecondsRef.current.delete(accountId); + const mediaCount = accountMediaCountRef.current.get(accountId) || 0; + setMultipleAccounts((prev) => prev.map((acc) => { + if (acc.id === accountId) { + const status = mediaCount === 0 ? ("failed" as const) : ("incomplete" as const); + return { ...acc, status, remainingTime: 0, mediaCount }; + } + return acc; + })); + CleanupExtractorProcesses().catch(() => { }); + } + } + }, 1000); + accountTimersRef.current.set(accountId, timerInterval); + try { + if (stopAllRef.current) { + clearInterval(timerInterval); + accountTimersRef.current.delete(accountId); + accountStartTimesRef.current.delete(accountId); + accountTimeoutSecondsRef.current.delete(accountId); + setMultipleAccounts((prev) => prev.map((acc) => { + if (acc.id === accountId) { + const status = acc.mediaCount === 0 ? ("failed" as const) : ("incomplete" as const); + return { ...acc, status }; + } + return acc; + })); + continue; + } + const fetchSettings = getSettings(); + const isSingleModeMultiple = fetchSettings.fetchMode === "single"; + const batchSizeMultiple = isSingleModeMultiple ? 0 : BATCH_SIZE; + const cleanUsername = account.username.trim(); + let allTimeline: TwitterResponse["timeline"] = []; + let accountInfo: TwitterResponse["account_info"] | null = null; + let cursor: string | undefined; + let hasMore = true; + let page = 0; + let previousMediaCount = account.mediaCount || 0; + while (hasMore && !stopAllRef.current) { + if (accountStopFlagsRef.current.get(accountId)) { + if (accountInfo && allTimeline.length > 0) { + saveFetchState({ + username: cleanUsername, + cursor: cursor || "", + timeline: allTimeline, + accountInfo: accountInfo, + totalFetched: allTimeline.length, + completed: false, + authToken: authToken.trim(), + mediaType: fetchedMediaType || "all", + retweets: false, + timelineType: "timeline", + }); + try { + await SaveAccountToDBWithStatus(accountInfo.name, accountInfo.nick, accountInfo.profile_image, allTimeline.length, JSON.stringify({ + account_info: accountInfo, + timeline: allTimeline, + total_urls: allTimeline.length, + metadata: { + new_entries: 0, + page: page, + batch_size: batchSizeMultiple, + has_more: hasMore, + cursor: cursor, + completed: false, + }, + cursor: cursor, + completed: false, + }), fetchedMediaType || "all", cursor || "", false); + } + catch (err) { + reportDatabaseSaveError("multi-account timeout state", err); + } + } + break; + } + const response = await ExtractTimeline({ + username: cleanUsername, + auth_token: authToken.trim(), + timeline_type: "timeline", + batch_size: batchSizeMultiple, + page: page, + media_type: fetchedMediaType || "all", + retweets: false, + cursor: cursor, + }); + const data: TwitterResponse = JSON.parse(response); + if (!accountInfo && data.account_info) { + accountInfo = data.account_info; + } + allTimeline = mergeTimelines(allTimeline, data.timeline); + cursor = data.cursor; + hasMore = !!data.cursor && !data.completed; + page++; + if (cursor) { + saveCursor(cleanUsername, cursor); + } + const currentMediaCount = allTimeline.length; + accountMediaCountRef.current.set(accountId, currentMediaCount); + const hasNewItems = currentMediaCount > previousMediaCount; + setMultipleAccounts((prev) => prev.map((acc) => { + if (acc.id === accountId) { + return { + ...acc, + accountInfo: accountInfo || undefined, + previousMediaCount: previousMediaCount, + mediaCount: currentMediaCount, + showDiff: hasNewItems, + cursor: cursor, + }; + } + return acc; + })); + if (hasNewItems) { + setTimeout(() => { + setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId ? { ...acc, showDiff: false } : acc)); + }, 1000); + } + previousMediaCount = currentMediaCount; + if (accountInfo) { + saveFetchState({ + username: cleanUsername, + cursor: cursor || "", + timeline: allTimeline, + accountInfo: accountInfo, + totalFetched: allTimeline.length, + completed: !hasMore, + authToken: authToken.trim(), + mediaType: fetchedMediaType || "all", + retweets: false, + timelineType: "timeline", + }); + } + if (accountInfo) { + try { + await SaveAccountToDBWithStatus(accountInfo.name, accountInfo.nick, accountInfo.profile_image, allTimeline.length, JSON.stringify({ + account_info: accountInfo, + timeline: allTimeline, + total_urls: allTimeline.length, + metadata: { + new_entries: data.timeline.length, + page: page, + batch_size: batchSizeMultiple, + has_more: hasMore, + cursor: cursor, + completed: !hasMore, + }, + cursor: cursor, + completed: !hasMore, + }), fetchedMediaType || "all", cursor || "", !hasMore); + } + catch (err) { + reportDatabaseSaveError("multi-account progress", err); + } + } + } + const startTime = accountStartTimesRef.current.get(accountId); + const elapsedSecs = startTime ? Math.floor((Date.now() - startTime) / 1000) : 0; + clearInterval(timerInterval); + accountTimersRef.current.delete(accountId); + accountStartTimesRef.current.delete(accountId); + accountTimeoutSecondsRef.current.delete(accountId); + const finalMediaCount = allTimeline.length; + accountMediaCountRef.current.set(accountId, finalMediaCount); + const wasTimeout = accountStopFlagsRef.current.get(accountId); + if (wasTimeout) { + logger.warning(`@${account.username}: timeout - ${finalMediaCount} items (${elapsedSecs}s)`); + setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId + ? { ...acc, status: finalMediaCount === 0 ? ("failed" as const) : ("incomplete" as const) } + : acc)); + } + else if (stopAllRef.current) { + logger.info(`@${account.username}: stopped - ${finalMediaCount} items (${elapsedSecs}s)`); + setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId + ? { ...acc, status: finalMediaCount === 0 ? ("failed" as const) : ("incomplete" as const) } + : acc)); + } + else if (hasMore) { + logger.warning(`@${account.username}: incomplete - ${finalMediaCount} items (${elapsedSecs}s)`); + setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId + ? { ...acc, status: finalMediaCount === 0 ? ("failed" as const) : ("incomplete" as const) } + : acc)); + } + else { + logger.success(`@${account.username}: completed - ${finalMediaCount} items (${elapsedSecs}s)`); + setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId + ? { ...acc, status: "completed" as const } + : acc)); + } + } + catch (error) { + const startTime = accountStartTimesRef.current.get(accountId); + const elapsedSecs = startTime ? Math.floor((Date.now() - startTime) / 1000) : 0; + const timerInterval = accountTimersRef.current.get(accountId); + if (timerInterval) { + clearInterval(timerInterval); + accountTimersRef.current.delete(accountId); + } + accountStartTimesRef.current.delete(accountId); + accountTimeoutSecondsRef.current.delete(accountId); + const errorMsg = error instanceof Error ? error.message : String(error); + const mediaCount = accountMediaCountRef.current.get(accountId) || 0; + const isAuthError = errorMsg.toLowerCase().includes("401") || + errorMsg.toLowerCase().includes("unauthorized") || + errorMsg.toLowerCase().includes("auth token may be invalid") || + errorMsg.toLowerCase().includes("auth token may be expired") || + errorMsg.toLowerCase().includes("invalid or expired"); + const status = isAuthError || mediaCount === 0 ? ("failed" as const) : ("incomplete" as const); + accountMediaCountRef.current.delete(accountId); + setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId + ? { + ...acc, + status, + error: errorMsg, + mediaCount, + } + : acc)); + logger.error(`@${account.username}: failed - ${errorMsg} (${elapsedSecs}s)`); + } + } + setIsFetchingAll(false); + if (!stopAllRef.current) { + toast.success("All accounts fetched"); + } }; - input.click(); - }; - - // Handle fetch all accounts - const handleFetchAll = async () => { - if (multipleAccounts.length === 0) { - toast.error("No accounts to fetch"); - return; - } - - setIsFetchingAll(true); - stopAllRef.current = false; - - // Get auth token from localStorage (same as SearchBar) - const PUBLIC_AUTH_TOKEN_KEY = "twitter_public_auth_token"; - const authToken = localStorage.getItem(PUBLIC_AUTH_TOKEN_KEY) || ""; - if (!authToken.trim()) { - toast.error("Please enter your auth token"); - setIsFetchingAll(false); - return; - } - - // Get timeout from settings - const settings = getSettings(); - const timeoutSeconds = settings.fetchTimeout || 60; - - // Reset all accounts to pending - setMultipleAccounts((prev) => - prev.map((acc) => ({ - ...acc, - status: "pending" as const, - mediaCount: 0, - previousMediaCount: 0, - elapsedTime: 0, - remainingTime: timeoutSeconds, - error: undefined, - showDiff: false, - })) - ); - - // Get current accounts list (use state getter function) - let currentAccounts = [...multipleAccounts]; - - // Fetch each account sequentially - for (let i = 0; i < currentAccounts.length; i++) { - if (stopAllRef.current) { - break; - } - - const account = currentAccounts[i]; - const accountId = account.id; - accountStopFlagsRef.current.set(accountId, false); - - // Update status to fetching - setMultipleAccounts((prev) => - prev.map((acc) => - acc.id === accountId - ? { ...acc, status: "fetching" as const } - : acc - ) - ); - - // Start timer for this account (countdown) - const settings = getSettings(); - const timeoutSeconds = settings.fetchTimeout || 60; - accountStartTimesRef.current.set(accountId, Date.now()); - accountTimeoutSecondsRef.current.set(accountId, timeoutSeconds); - - // Initialize with timeout - setMultipleAccounts((prev) => - prev.map((acc) => - acc.id === accountId + const handleStopAll = () => { + stopAllRef.current = true; + setIsFetchingAll(false); + accountTimersRef.current.forEach((interval) => clearInterval(interval)); + accountTimersRef.current.clear(); + accountStartTimesRef.current.clear(); + accountTimeoutSecondsRef.current.clear(); + setMultipleAccounts((prev) => prev.map((acc) => { + if (acc.status === "fetching") { + const mediaCount = accountMediaCountRef.current.get(acc.id) || acc.mediaCount || 0; + const status = mediaCount === 0 ? ("failed" as const) : ("incomplete" as const); + return { ...acc, status, mediaCount }; + } + return acc; + })); + CleanupExtractorProcesses().catch(() => { }); + toast.info("Stopped all fetches"); + }; + const handleStopAccount = (accountId: string) => { + accountStopFlagsRef.current.set(accountId, true); + setMultipleAccounts((prev) => prev.map((acc) => { + if (acc.id === accountId && acc.status === "fetching") { + const mediaCount = accountMediaCountRef.current.get(accountId) || acc.mediaCount || 0; + const status = mediaCount === 0 ? ("failed" as const) : ("incomplete" as const); + return { ...acc, status, mediaCount }; + } + return acc; + })); + const timerInterval = accountTimersRef.current.get(accountId); + if (timerInterval) { + clearInterval(timerInterval); + accountTimersRef.current.delete(accountId); + } + accountStartTimesRef.current.delete(accountId); + accountTimeoutSecondsRef.current.delete(accountId); + CleanupExtractorProcesses().catch(() => { }); + toast.info("Stopped account fetch"); + }; + const handleRetryAccount = async (accountId: string) => { + const account = multipleAccounts.find((acc) => acc.id === accountId); + if (!account) + return; + const PUBLIC_AUTH_TOKEN_KEY = "twitter_public_auth_token"; + const authToken = localStorage.getItem(PUBLIC_AUTH_TOKEN_KEY) || ""; + if (!authToken.trim()) { + toast.error("Please enter your auth token"); + return; + } + const retrySettings = getSettings(); + const isSingleModeRetry = retrySettings.fetchMode === "single"; + const batchSizeRetry = isSingleModeRetry ? 0 : BATCH_SIZE; + const cleanUsername = account.username.trim(); + let existingState: FetchState | null = null; + let cursor: string | undefined = account.cursor; + let allTimeline: TwitterResponse["timeline"] = []; + let accountInfo: TwitterResponse["account_info"] | null = null; + let previousMediaCount = account.mediaCount || 0; + existingState = getFetchState(cleanUsername); + if (existingState && existingState.cursor && !existingState.completed) { + cursor = existingState.cursor; + allTimeline = existingState.timeline; + if (existingState.accountInfo) { + accountInfo = existingState.accountInfo; + } + previousMediaCount = existingState.timeline.length; + logger.info(`Resuming @${cleanUsername} from ${allTimeline.length} items...`); + } + else { + if (!cursor) { + const savedCursor = getCursor(cleanUsername); + if (savedCursor) { + cursor = savedCursor; + previousMediaCount = account.mediaCount || 0; + logger.info(`Retrying @${cleanUsername} from cursor (${previousMediaCount} items)...`); + } + else { + try { + const accounts = await GetAllAccountsFromDB(); + const dbAccount = accounts.find((acc) => acc.username.toLowerCase() === cleanUsername.toLowerCase()); + if (dbAccount?.cursor) { + cursor = dbAccount.cursor; + previousMediaCount = account.mediaCount || 0; + logger.info(`Retrying @${cleanUsername} from database cursor (${previousMediaCount} items)...`); + } + else { + clearFetchState(cleanUsername); + clearCursor(cleanUsername); + logger.info(`Retrying @${cleanUsername} from beginning...`); + } + } + catch (dbError) { + console.error("Failed to get cursor from database:", dbError); + clearFetchState(cleanUsername); + clearCursor(cleanUsername); + logger.info(`Retrying @${cleanUsername} from beginning...`); + } + } + } + else { + previousMediaCount = account.mediaCount || 0; + logger.info(`Retrying @${cleanUsername} from cursor (${previousMediaCount} items)...`); + } + } + setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId + ? { + ...acc, + status: "fetching" as const, + elapsedTime: 0, + remainingTime: null, + error: undefined, + showDiff: false, + mediaCount: previousMediaCount, + previousMediaCount: previousMediaCount, + } + : acc)); + accountStopFlagsRef.current.set(accountId, false); + const settings = getSettings(); + const timeoutSeconds = settings.fetchTimeout || 60; + accountStartTimesRef.current.set(accountId, Date.now()); + accountTimeoutSecondsRef.current.set(accountId, timeoutSeconds); + setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId ? { ...acc, elapsedTime: 0, remainingTime: timeoutSeconds } - : acc - ) - ); - - const timerInterval = window.setInterval(() => { - const startTime = accountStartTimesRef.current.get(accountId); - const timeoutSecs = accountTimeoutSecondsRef.current.get(accountId) || timeoutSeconds; - if (startTime) { - const elapsed = Math.floor((Date.now() - startTime) / 1000); - const remaining = Math.max(0, timeoutSecs - elapsed); - setMultipleAccounts((prev) => - prev.map((acc) => - acc.id === accountId - ? { ...acc, elapsedTime: elapsed, remainingTime: remaining } - : acc - ) - ); - - // Auto-stop if timeout reached - if (remaining <= 0) { - accountStopFlagsRef.current.set(accountId, true); + : acc)); + const timerInterval = window.setInterval(() => { + const startTime = accountStartTimesRef.current.get(accountId); + const timeoutSecs = accountTimeoutSecondsRef.current.get(accountId) || timeoutSeconds; + if (startTime) { + const elapsed = Math.floor((Date.now() - startTime) / 1000); + const remaining = Math.max(0, timeoutSecs - elapsed); + setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId + ? { ...acc, elapsedTime: elapsed, remainingTime: remaining } + : acc)); + if (remaining <= 0) { + accountStopFlagsRef.current.set(accountId, true); + clearInterval(timerInterval); + accountTimersRef.current.delete(accountId); + accountStartTimesRef.current.delete(accountId); + accountTimeoutSecondsRef.current.delete(accountId); + const mediaCount = accountMediaCountRef.current.get(accountId) || 0; + setMultipleAccounts((prev) => prev.map((acc) => { + if (acc.id === accountId) { + const status = mediaCount === 0 ? ("failed" as const) : ("incomplete" as const); + return { ...acc, status, remainingTime: 0, mediaCount }; + } + return acc; + })); + CleanupExtractorProcesses().catch(() => { }); + } + } + }, 1000); + accountTimersRef.current.set(accountId, timerInterval); + try { + let hasMore = true; + let page = 0; + while (hasMore && !stopAllRef.current) { + if (accountStopFlagsRef.current.get(accountId)) { + break; + } + const response = await ExtractTimeline({ + username: cleanUsername, + auth_token: authToken.trim(), + timeline_type: "timeline", + batch_size: batchSizeRetry, + page: page, + media_type: fetchedMediaType || "all", + retweets: false, + cursor: cursor, + }); + const data: TwitterResponse = JSON.parse(response); + if (!accountInfo && data.account_info) { + accountInfo = data.account_info; + } + allTimeline = mergeTimelines(allTimeline, data.timeline); + cursor = data.cursor; + hasMore = !!data.cursor && !data.completed; + page++; + if (cursor) { + saveCursor(cleanUsername, cursor); + } + const currentMediaCount = allTimeline.length; + accountMediaCountRef.current.set(accountId, currentMediaCount); + const hasNewItems = currentMediaCount > previousMediaCount; + setMultipleAccounts((prev) => prev.map((acc) => { + if (acc.id === accountId) { + return { + ...acc, + accountInfo: accountInfo || undefined, + previousMediaCount: previousMediaCount, + mediaCount: currentMediaCount, + showDiff: hasNewItems, + cursor: cursor, + }; + } + return acc; + })); + if (page % 3 === 0 || !hasMore || accountStopFlagsRef.current.get(accountId)) { + saveFetchState({ + username: cleanUsername, + cursor: cursor || "", + timeline: allTimeline, + accountInfo: accountInfo, + totalFetched: allTimeline.length, + completed: !hasMore, + authToken: authToken.trim(), + mediaType: fetchedMediaType || "all", + retweets: false, + timelineType: "timeline", + }); + } + if (hasNewItems) { + setTimeout(() => { + setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId ? { ...acc, showDiff: false } : acc)); + }, 1000); + } + previousMediaCount = currentMediaCount; + if (accountInfo) { + try { + await SaveAccountToDBWithStatus(accountInfo.name, accountInfo.nick, accountInfo.profile_image, allTimeline.length, JSON.stringify({ + account_info: accountInfo, + timeline: allTimeline, + total_urls: allTimeline.length, + metadata: { + new_entries: data.timeline.length, + page: page, + batch_size: batchSizeRetry, + has_more: hasMore, + cursor: cursor, + completed: !hasMore, + }, + cursor: cursor, + completed: !hasMore, + }), fetchedMediaType || "all", cursor || "", !hasMore); + } + catch (err) { + reportDatabaseSaveError("retry progress", err); + } + } + } + const startTime = accountStartTimesRef.current.get(accountId); + const elapsedSecs = startTime ? Math.floor((Date.now() - startTime) / 1000) : 0; clearInterval(timerInterval); accountTimersRef.current.delete(accountId); accountStartTimesRef.current.delete(accountId); accountTimeoutSecondsRef.current.delete(accountId); - - // Check if any media was fetched using ref (most up-to-date value) + const finalMediaCount = allTimeline.length; + accountMediaCountRef.current.set(accountId, finalMediaCount); + const wasTimeout = accountStopFlagsRef.current.get(accountId); + if (wasTimeout) { + logger.warning(`@${account.username}: timeout - ${finalMediaCount} items (${elapsedSecs}s)`); + setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId + ? { ...acc, status: finalMediaCount === 0 ? ("failed" as const) : ("incomplete" as const) } + : acc)); + } + else if (stopAllRef.current) { + logger.info(`@${account.username}: stopped - ${finalMediaCount} items (${elapsedSecs}s)`); + setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId + ? { ...acc, status: finalMediaCount === 0 ? ("failed" as const) : ("incomplete" as const) } + : acc)); + } + else if (hasMore) { + logger.warning(`@${account.username}: incomplete - ${finalMediaCount} items (${elapsedSecs}s)`); + setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId + ? { ...acc, status: finalMediaCount === 0 ? ("failed" as const) : ("incomplete" as const) } + : acc)); + } + else { + logger.success(`@${account.username}: completed - ${finalMediaCount} items (${elapsedSecs}s)`); + clearFetchState(cleanUsername); + clearCursor(cleanUsername); + setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId ? { ...acc, status: "completed" as const, cursor: undefined } : acc)); + } + } + catch (error) { + const startTime = accountStartTimesRef.current.get(accountId); + const elapsedSecs = startTime ? Math.floor((Date.now() - startTime) / 1000) : 0; + const timerInterval = accountTimersRef.current.get(accountId); + if (timerInterval) { + clearInterval(timerInterval); + accountTimersRef.current.delete(accountId); + } + accountStartTimesRef.current.delete(accountId); + accountTimeoutSecondsRef.current.delete(accountId); + const errorMsg = error instanceof Error ? error.message : String(error); const mediaCount = accountMediaCountRef.current.get(accountId) || 0; - setMultipleAccounts((prev) => - prev.map((acc) => { - if (acc.id === accountId) { - // If no media was fetched at all, mark as failed - // Otherwise mark as incomplete - const status = mediaCount === 0 ? ("failed" as const) : ("incomplete" as const); - return { ...acc, status, remainingTime: 0, mediaCount }; + const isAuthError = errorMsg.toLowerCase().includes("401") || + errorMsg.toLowerCase().includes("unauthorized") || + errorMsg.toLowerCase().includes("auth token may be invalid") || + errorMsg.toLowerCase().includes("auth token may be expired") || + errorMsg.toLowerCase().includes("invalid or expired"); + const status = isAuthError || mediaCount === 0 ? ("failed" as const) : ("incomplete" as const); + accountMediaCountRef.current.delete(accountId); + setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId + ? { + ...acc, + status, + error: errorMsg, + mediaCount, } - return acc; - }) - ); - CleanupExtractorProcesses().catch(() => {}); - } + : acc)); + logger.error(`@${account.username}: failed - ${errorMsg} (${elapsedSecs}s)`); } - }, 1000); - accountTimersRef.current.set(accountId, timerInterval); + }; + const renderPage = () => { + switch (currentPage) { + case "settings": + return ; + case "debug": + return ; + case "database": + return ( setCurrentPage("main")} onLoadAccount={handleLoadFromDB} onUpdateSelected={handleUpdateSelected}/>); + default: + return (<> +
- try { - // Check if account should be stopped - if (stopAllRef.current) { - clearInterval(timerInterval); - accountTimersRef.current.delete(accountId); - accountStartTimesRef.current.delete(accountId); - accountTimeoutSecondsRef.current.delete(accountId); - // Check if any media was fetched - setMultipleAccounts((prev) => - prev.map((acc) => { - if (acc.id === accountId) { - const status = acc.mediaCount === 0 ? ("failed" as const) : ("incomplete" as const); - return { ...acc, status }; - } - return acc; - }) - ); - continue; + { + setSearchMode(mode); + if (privateType) { + setSearchPrivateType(privateType); + } + }}/> + + {result && fetchType === "single" && (
+ +
)} + ); } - - // Fetch account - check fetch mode from settings - const fetchSettings = getSettings(); - const isSingleModeMultiple = fetchSettings.fetchMode === "single"; - const batchSizeMultiple = isSingleModeMultiple ? 0 : BATCH_SIZE; - - const cleanUsername = account.username.trim(); - let allTimeline: TwitterResponse["timeline"] = []; - let accountInfo: TwitterResponse["account_info"] | null = null; - let cursor: string | undefined; - let hasMore = true; - let page = 0; - let previousMediaCount = account.mediaCount || 0; - - while (hasMore && !stopAllRef.current) { - // Check if this specific account should be stopped (timeout) - if (accountStopFlagsRef.current.get(accountId)) { - // IMPORTANT: Save state before breaking to prevent data loss - if (accountInfo && allTimeline.length > 0) { - saveFetchState({ - username: cleanUsername, - cursor: cursor || "", - timeline: allTimeline, - accountInfo: accountInfo, - totalFetched: allTimeline.length, - completed: false, - authToken: authToken.trim(), - mediaType: fetchedMediaType || "all", - retweets: false, - timelineType: "timeline", - }); - - // Also save to database - try { - await SaveAccountToDBWithStatus( - accountInfo.name, - accountInfo.nick, - accountInfo.profile_image, - allTimeline.length, - JSON.stringify({ - account_info: accountInfo, - timeline: allTimeline, - total_urls: allTimeline.length, - metadata: { - new_entries: 0, - page: page, - batch_size: batchSizeMultiple, - has_more: hasMore, - cursor: cursor, - completed: false, - }, - cursor: cursor, - completed: false, - }), - fetchedMediaType || "all", - cursor || "", - false - ); - } catch (err) { - console.error("Failed to save timeout state to database:", err); - } - } - break; - } - - const response = await ExtractTimeline({ - username: cleanUsername, - auth_token: authToken.trim(), - timeline_type: "timeline", - batch_size: batchSizeMultiple, - page: page, - media_type: fetchedMediaType || "all", - retweets: false, - cursor: cursor, - }); - - const data: TwitterResponse = JSON.parse(response); - - if (!accountInfo && data.account_info) { - accountInfo = data.account_info; - } - - allTimeline = mergeTimelines(allTimeline, data.timeline); - - cursor = data.cursor; - hasMore = !!data.cursor && !data.completed; - page++; - - // Save cursor every batch (lightweight, just a string) - if (cursor) { - saveCursor(cleanUsername, cursor); - } - - // Update account with progress - const currentMediaCount = allTimeline.length; - accountMediaCountRef.current.set(accountId, currentMediaCount); - const hasNewItems = currentMediaCount > previousMediaCount; - - setMultipleAccounts((prev) => - prev.map((acc) => { - if (acc.id === accountId) { - return { - ...acc, - accountInfo: accountInfo || undefined, - previousMediaCount: previousMediaCount, - mediaCount: currentMediaCount, - showDiff: hasNewItems, - cursor: cursor, // Save cursor for retry - }; - } - return acc; - }) - ); - - // Clear diff after 1 second - if (hasNewItems) { - setTimeout(() => { - setMultipleAccounts((prev) => - prev.map((acc) => - acc.id === accountId ? { ...acc, showDiff: false } : acc - ) - ); - }, 1000); - } - - // Update previous count for next iteration - previousMediaCount = currentMediaCount; - - // Save fetch state EVERY batch to ensure no data loss on timeout/error - if (accountInfo) { - saveFetchState({ - username: cleanUsername, - cursor: cursor || "", - timeline: allTimeline, - accountInfo: accountInfo, - totalFetched: allTimeline.length, - completed: !hasMore, - authToken: authToken.trim(), - mediaType: fetchedMediaType || "all", - retweets: false, - timelineType: "timeline", - }); - } - - // Save to database after each batch - if (accountInfo) { - try { - await SaveAccountToDBWithStatus( - accountInfo.name, - accountInfo.nick, - accountInfo.profile_image, - allTimeline.length, - JSON.stringify({ - account_info: accountInfo, - timeline: allTimeline, - total_urls: allTimeline.length, - metadata: { - new_entries: data.timeline.length, - page: page, - batch_size: batchSizeMultiple, - has_more: hasMore, - cursor: cursor, - completed: !hasMore, - }, - cursor: cursor, - completed: !hasMore, - }), - fetchedMediaType || "all", - cursor || "", - !hasMore - ); - } catch (err) { - console.error("Failed to save progress to database:", err); - } - } - } - - // Calculate elapsed time before clearing - const startTime = accountStartTimesRef.current.get(accountId); - const elapsedSecs = startTime ? Math.floor((Date.now() - startTime) / 1000) : 0; - - // Clear timer - clearInterval(timerInterval); - accountTimersRef.current.delete(accountId); - accountStartTimesRef.current.delete(accountId); - accountTimeoutSecondsRef.current.delete(accountId); - - // Update final status - check if media was fetched - const finalMediaCount = allTimeline.length; - accountMediaCountRef.current.set(accountId, finalMediaCount); - - // Check if stopped due to timeout (accountStopFlags was set by timeout handler) - const wasTimeout = accountStopFlagsRef.current.get(accountId); - - if (wasTimeout) { - // Timeout: incomplete if media was fetched, failed if no media - logger.warning(`@${account.username}: timeout - ${finalMediaCount} items (${elapsedSecs}s)`); - setMultipleAccounts((prev) => - prev.map((acc) => - acc.id === accountId - ? { ...acc, status: finalMediaCount === 0 ? ("failed" as const) : ("incomplete" as const) } - : acc - ) - ); - } else if (stopAllRef.current) { - // Stopped by user: incomplete if media was fetched, failed if no media - logger.info(`@${account.username}: stopped - ${finalMediaCount} items (${elapsedSecs}s)`); - setMultipleAccounts((prev) => - prev.map((acc) => - acc.id === accountId - ? { ...acc, status: finalMediaCount === 0 ? ("failed" as const) : ("incomplete" as const) } - : acc - ) - ); - } else if (hasMore) { - // Has more but stopped (e.g., API error): incomplete if media was fetched, failed if no media - logger.warning(`@${account.username}: incomplete - ${finalMediaCount} items (${elapsedSecs}s)`); - setMultipleAccounts((prev) => - prev.map((acc) => - acc.id === accountId - ? { ...acc, status: finalMediaCount === 0 ? ("failed" as const) : ("incomplete" as const) } - : acc - ) - ); - } else { - // Completed successfully - logger.success(`@${account.username}: completed - ${finalMediaCount} items (${elapsedSecs}s)`); - setMultipleAccounts((prev) => - prev.map((acc) => - acc.id === accountId - ? { ...acc, status: "completed" as const } - : acc - ) - ); - } - } catch (error) { - // Calculate elapsed time before clearing - const startTime = accountStartTimesRef.current.get(accountId); - const elapsedSecs = startTime ? Math.floor((Date.now() - startTime) / 1000) : 0; - - // Clear timer on error - const timerInterval = accountTimersRef.current.get(accountId); - if (timerInterval) { - clearInterval(timerInterval); - accountTimersRef.current.delete(accountId); - } - accountStartTimesRef.current.delete(accountId); - accountTimeoutSecondsRef.current.delete(accountId); - - const errorMsg = error instanceof Error ? error.message : String(error); - - // Check if any media was fetched before the error - const mediaCount = accountMediaCountRef.current.get(accountId) || 0; - - // Check if error is auth token related (401, unauthorized, invalid, expired) - const isAuthError = errorMsg.toLowerCase().includes("401") || - errorMsg.toLowerCase().includes("unauthorized") || - errorMsg.toLowerCase().includes("auth token may be invalid") || - errorMsg.toLowerCase().includes("auth token may be expired") || - errorMsg.toLowerCase().includes("invalid or expired"); - - // Determine status: - // - If auth error: always "failed" (even if media was fetched, it's likely incomplete data) - // - If not auth error and media was fetched: "incomplete" - // - If not auth error and no media: "failed" - const status = isAuthError || mediaCount === 0 ? ("failed" as const) : ("incomplete" as const); - - accountMediaCountRef.current.delete(accountId); - - setMultipleAccounts((prev) => - prev.map((acc) => - acc.id === accountId - ? { - ...acc, - status, - error: errorMsg, - mediaCount, - } - : acc - ) - ); - logger.error(`@${account.username}: failed - ${errorMsg} (${elapsedSecs}s)`); - } - } - - setIsFetchingAll(false); - if (!stopAllRef.current) { - toast.success("All accounts fetched"); - } - }; - - // Handle stop all - const handleStopAll = () => { - stopAllRef.current = true; - setIsFetchingAll(false); - - // Stop all timers - accountTimersRef.current.forEach((interval) => clearInterval(interval)); - accountTimersRef.current.clear(); - accountStartTimesRef.current.clear(); - accountTimeoutSecondsRef.current.clear(); - // Keep accountMediaCountRef for status checking - - // Update all fetching accounts - check if they have media using ref (most up-to-date value) - setMultipleAccounts((prev) => - prev.map((acc) => { - if (acc.status === "fetching") { - // If no media was fetched, mark as failed; otherwise incomplete - const mediaCount = accountMediaCountRef.current.get(acc.id) || acc.mediaCount || 0; - const status = mediaCount === 0 ? ("failed" as const) : ("incomplete" as const); - return { ...acc, status, mediaCount }; - } - return acc; - }) - ); - - // Cleanup processes - CleanupExtractorProcesses().catch(() => {}); - toast.info("Stopped all fetches"); - }; - - // Handle stop specific account - const handleStopAccount = (accountId: string) => { - // Set stop flag for this account - accountStopFlagsRef.current.set(accountId, true); - - // Check if any media was fetched before marking status using ref (most up-to-date value) - setMultipleAccounts((prev) => - prev.map((acc) => { - if (acc.id === accountId && acc.status === "fetching") { - // If no media was fetched, mark as failed; otherwise incomplete - const mediaCount = accountMediaCountRef.current.get(accountId) || acc.mediaCount || 0; - const status = mediaCount === 0 ? ("failed" as const) : ("incomplete" as const); - return { ...acc, status, mediaCount }; - } - return acc; - }) - ); - - // Clear timer for this account - const timerInterval = accountTimersRef.current.get(accountId); - if (timerInterval) { - clearInterval(timerInterval); - accountTimersRef.current.delete(accountId); - } - accountStartTimesRef.current.delete(accountId); - accountTimeoutSecondsRef.current.delete(accountId); - - // Cleanup processes - CleanupExtractorProcesses().catch(() => {}); - toast.info("Stopped account fetch"); - }; - - // Handle retry specific account - const handleRetryAccount = async (accountId: string) => { - const account = multipleAccounts.find((acc) => acc.id === accountId); - if (!account) return; - - // Get auth token from localStorage - const PUBLIC_AUTH_TOKEN_KEY = "twitter_public_auth_token"; - const authToken = localStorage.getItem(PUBLIC_AUTH_TOKEN_KEY) || ""; - if (!authToken.trim()) { - toast.error("Please enter your auth token"); - return; - } - - // Get fetch mode from settings - const retrySettings = getSettings(); - const isSingleModeRetry = retrySettings.fetchMode === "single"; - const batchSizeRetry = isSingleModeRetry ? 0 : BATCH_SIZE; - - // Check if we have a saved cursor (like resume in single fetch) - const cleanUsername = account.username.trim(); - let existingState: FetchState | null = null; - let cursor: string | undefined = account.cursor; // Use saved cursor from account state - let allTimeline: TwitterResponse["timeline"] = []; - let accountInfo: TwitterResponse["account_info"] | null = null; - let previousMediaCount = account.mediaCount || 0; - - // Try to load from fetch state (like single fetch resume) - existingState = getFetchState(cleanUsername); - if (existingState && existingState.cursor && !existingState.completed) { - cursor = existingState.cursor; - allTimeline = existingState.timeline; - if (existingState.accountInfo) { - accountInfo = existingState.accountInfo; - } - previousMediaCount = existingState.timeline.length; - logger.info(`Resuming @${cleanUsername} from ${allTimeline.length} items...`); - } else { - // Try to get cursor from lightweight storage (localStorage) - if (!cursor) { - const savedCursor = getCursor(cleanUsername); - if (savedCursor) { - cursor = savedCursor; - previousMediaCount = account.mediaCount || 0; - logger.info(`Retrying @${cleanUsername} from cursor (${previousMediaCount} items)...`); - } else { - // Fallback: try to get cursor from database - try { - const accounts = await GetAllAccountsFromDB(); - const dbAccount = accounts.find( - (acc) => acc.username.toLowerCase() === cleanUsername.toLowerCase() - ); - if (dbAccount?.cursor) { - cursor = dbAccount.cursor; - previousMediaCount = account.mediaCount || 0; - logger.info(`Retrying @${cleanUsername} from database cursor (${previousMediaCount} items)...`); - } else { - // Fresh retry - clear any existing state - clearFetchState(cleanUsername); - clearCursor(cleanUsername); - logger.info(`Retrying @${cleanUsername} from beginning...`); - } - } catch (dbError) { - console.error("Failed to get cursor from database:", dbError); - clearFetchState(cleanUsername); - clearCursor(cleanUsername); - logger.info(`Retrying @${cleanUsername} from beginning...`); - } - } - } else { - previousMediaCount = account.mediaCount || 0; - logger.info(`Retrying @${cleanUsername} from cursor (${previousMediaCount} items)...`); - } - } - - // Update account status to fetching (preserve existing data) - setMultipleAccounts((prev) => - prev.map((acc) => - acc.id === accountId - ? { - ...acc, - status: "fetching" as const, - elapsedTime: 0, - remainingTime: null, - error: undefined, - showDiff: false, - mediaCount: previousMediaCount, // Preserve existing count - previousMediaCount: previousMediaCount, - } - : acc - ) - ); - - accountStopFlagsRef.current.set(accountId, false); - - // Start timer for this account - const settings = getSettings(); - const timeoutSeconds = settings.fetchTimeout || 60; - accountStartTimesRef.current.set(accountId, Date.now()); - accountTimeoutSecondsRef.current.set(accountId, timeoutSeconds); - - setMultipleAccounts((prev) => - prev.map((acc) => - acc.id === accountId - ? { ...acc, elapsedTime: 0, remainingTime: timeoutSeconds } - : acc - ) - ); - - const timerInterval = window.setInterval(() => { - const startTime = accountStartTimesRef.current.get(accountId); - const timeoutSecs = accountTimeoutSecondsRef.current.get(accountId) || timeoutSeconds; - if (startTime) { - const elapsed = Math.floor((Date.now() - startTime) / 1000); - const remaining = Math.max(0, timeoutSecs - elapsed); - setMultipleAccounts((prev) => - prev.map((acc) => - acc.id === accountId - ? { ...acc, elapsedTime: elapsed, remainingTime: remaining } - : acc - ) - ); - - // Auto-stop if timeout reached - if (remaining <= 0) { - accountStopFlagsRef.current.set(accountId, true); - clearInterval(timerInterval); - accountTimersRef.current.delete(accountId); - accountStartTimesRef.current.delete(accountId); - accountTimeoutSecondsRef.current.delete(accountId); - - // Check if any media was fetched using ref (most up-to-date value) - const mediaCount = accountMediaCountRef.current.get(accountId) || 0; - setMultipleAccounts((prev) => - prev.map((acc) => { - if (acc.id === accountId) { - const status = mediaCount === 0 ? ("failed" as const) : ("incomplete" as const); - return { ...acc, status, remainingTime: 0, mediaCount }; - } - return acc; - }) - ); - CleanupExtractorProcesses().catch(() => {}); - } - } - }, 1000); - accountTimersRef.current.set(accountId, timerInterval); - - try { - let hasMore = true; - let page = 0; - - while (hasMore && !stopAllRef.current) { - // Check if this specific account should be stopped - if (accountStopFlagsRef.current.get(accountId)) { - break; - } - - const response = await ExtractTimeline({ - username: cleanUsername, - auth_token: authToken.trim(), - timeline_type: "timeline", - batch_size: batchSizeRetry, - page: page, - media_type: fetchedMediaType || "all", - retweets: false, - cursor: cursor, - }); - - const data: TwitterResponse = JSON.parse(response); - - if (!accountInfo && data.account_info) { - accountInfo = data.account_info; - } - - allTimeline = mergeTimelines(allTimeline, data.timeline); - - cursor = data.cursor; - hasMore = !!data.cursor && !data.completed; - page++; - - // Save cursor every batch (lightweight, just a string) - if (cursor) { - saveCursor(cleanUsername, cursor); - } - - // Update account with progress - const currentMediaCount = allTimeline.length; - accountMediaCountRef.current.set(accountId, currentMediaCount); - const hasNewItems = currentMediaCount > previousMediaCount; - - setMultipleAccounts((prev) => - prev.map((acc) => { - if (acc.id === accountId) { - return { - ...acc, - accountInfo: accountInfo || undefined, - previousMediaCount: previousMediaCount, - mediaCount: currentMediaCount, - showDiff: hasNewItems, - cursor: cursor, // Save cursor for next retry - }; - } - return acc; - }) - ); - - // Save state periodically (like single fetch) for resume capability - if (page % 3 === 0 || !hasMore || accountStopFlagsRef.current.get(accountId)) { - saveFetchState({ - username: cleanUsername, - cursor: cursor || "", - timeline: allTimeline, - accountInfo: accountInfo, - totalFetched: allTimeline.length, - completed: !hasMore, - authToken: authToken.trim(), - mediaType: fetchedMediaType || "all", - retweets: false, - timelineType: "timeline", - }); - } - - // Clear diff after 1 second - if (hasNewItems) { - setTimeout(() => { - setMultipleAccounts((prev) => - prev.map((acc) => - acc.id === accountId ? { ...acc, showDiff: false } : acc - ) - ); - }, 1000); - } - - // Update previous count for next iteration - previousMediaCount = currentMediaCount; - - // Save to database after each batch - if (accountInfo) { - try { - await SaveAccountToDBWithStatus( - accountInfo.name, - accountInfo.nick, - accountInfo.profile_image, - allTimeline.length, - JSON.stringify({ - account_info: accountInfo, - timeline: allTimeline, - total_urls: allTimeline.length, - metadata: { - new_entries: data.timeline.length, - page: page, - batch_size: batchSizeRetry, - has_more: hasMore, - cursor: cursor, - completed: !hasMore, - }, - cursor: cursor, - completed: !hasMore, - }), - fetchedMediaType || "all", - cursor || "", - !hasMore - ); - } catch (err) { - console.error("Failed to save progress to database:", err); - } - } - } - - // Calculate elapsed time before clearing - const startTime = accountStartTimesRef.current.get(accountId); - const elapsedSecs = startTime ? Math.floor((Date.now() - startTime) / 1000) : 0; - - // Clear timer - clearInterval(timerInterval); - accountTimersRef.current.delete(accountId); - accountStartTimesRef.current.delete(accountId); - accountTimeoutSecondsRef.current.delete(accountId); - - // Update final status - const finalMediaCount = allTimeline.length; - accountMediaCountRef.current.set(accountId, finalMediaCount); - - const wasTimeout = accountStopFlagsRef.current.get(accountId); - - if (wasTimeout) { - logger.warning(`@${account.username}: timeout - ${finalMediaCount} items (${elapsedSecs}s)`); - setMultipleAccounts((prev) => - prev.map((acc) => - acc.id === accountId - ? { ...acc, status: finalMediaCount === 0 ? ("failed" as const) : ("incomplete" as const) } - : acc - ) - ); - } else if (stopAllRef.current) { - logger.info(`@${account.username}: stopped - ${finalMediaCount} items (${elapsedSecs}s)`); - setMultipleAccounts((prev) => - prev.map((acc) => - acc.id === accountId - ? { ...acc, status: finalMediaCount === 0 ? ("failed" as const) : ("incomplete" as const) } - : acc - ) - ); - } else if (hasMore) { - logger.warning(`@${account.username}: incomplete - ${finalMediaCount} items (${elapsedSecs}s)`); - setMultipleAccounts((prev) => - prev.map((acc) => - acc.id === accountId - ? { ...acc, status: finalMediaCount === 0 ? ("failed" as const) : ("incomplete" as const) } - : acc - ) - ); - } else { - // Completed successfully - clear fetch state and cursor - logger.success(`@${account.username}: completed - ${finalMediaCount} items (${elapsedSecs}s)`); - clearFetchState(cleanUsername); - clearCursor(cleanUsername); - setMultipleAccounts((prev) => - prev.map((acc) => - acc.id === accountId ? { ...acc, status: "completed" as const, cursor: undefined } : acc - ) - ); - } - } catch (error) { - // Calculate elapsed time before clearing - const startTime = accountStartTimesRef.current.get(accountId); - const elapsedSecs = startTime ? Math.floor((Date.now() - startTime) / 1000) : 0; - - // Clear timer on error - const timerInterval = accountTimersRef.current.get(accountId); - if (timerInterval) { - clearInterval(timerInterval); - accountTimersRef.current.delete(accountId); - } - accountStartTimesRef.current.delete(accountId); - accountTimeoutSecondsRef.current.delete(accountId); - - const errorMsg = error instanceof Error ? error.message : String(error); - - // Check if any media was fetched before the error - const mediaCount = accountMediaCountRef.current.get(accountId) || 0; - - // Check if error is auth token related - const isAuthError = errorMsg.toLowerCase().includes("401") || - errorMsg.toLowerCase().includes("unauthorized") || - errorMsg.toLowerCase().includes("auth token may be invalid") || - errorMsg.toLowerCase().includes("auth token may be expired") || - errorMsg.toLowerCase().includes("invalid or expired"); - - const status = isAuthError || mediaCount === 0 ? ("failed" as const) : ("incomplete" as const); - - accountMediaCountRef.current.delete(accountId); - - setMultipleAccounts((prev) => - prev.map((acc) => - acc.id === accountId - ? { - ...acc, - status, - error: errorMsg, - mediaCount, - } - : acc - ) - ); - logger.error(`@${account.username}: failed - ${errorMsg} (${elapsedSecs}s)`); - } - }; - - const renderPage = () => { - switch (currentPage) { - case "settings": - return ; - case "debug": - return ; - case "database": - return ( - setCurrentPage("main")} - onLoadAccount={handleLoadFromDB} - onUpdateSelected={handleUpdateSelected} - /> - ); - default: - return ( - <> -
- - { - setSearchMode(mode); - if (privateType) { - setSearchPrivateType(privateType); - } - }} - /> - - {result && fetchType === "single" && ( -
- -
- )} - - ); - } - }; - - return ( - + }; + return (
+ - + + - {/* Main content area with sidebar offset */}
{renderPage()}
-
- ); +
); } - export default App; diff --git a/frontend/src/components/DatabaseView.tsx b/frontend/src/components/DatabaseView.tsx index 28dd10a..3524837 100644 --- a/frontend/src/components/DatabaseView.tsx +++ b/frontend/src/components/DatabaseView.tsx @@ -5,784 +5,672 @@ import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { Progress } from "@/components/ui/progress"; import { cn } from "@/lib/utils"; -import { - Dialog, - DialogContent, - DialogDescription, - DialogFooter, - DialogHeader, - DialogTitle, - DialogTrigger, -} from "@/components/ui/dialog"; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from "@/components/ui/select"; -import { - Tooltip, - TooltipContent, - TooltipTrigger, -} from "@/components/ui/tooltip"; +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle, DialogTrigger, } from "@/components/ui/dialog"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue, } from "@/components/ui/select"; +import { Tooltip, TooltipContent, TooltipTrigger, } from "@/components/ui/tooltip"; import { Checkbox } from "@/components/ui/checkbox"; import { Spinner } from "@/components/ui/spinner"; -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuTrigger, -} from "@/components/ui/dropdown-menu"; +import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { Trash2, FileInput, FileOutput, Pencil, Tag, Shuffle, X, XCircle, Download, StopCircle, Globe, Lock, Bookmark, Heart, Image, Images, Video, Film, FileText, Filter, AlertCircle, MoreVertical, FileBraces, CloudBackup, Search, LayoutGrid, Grid3X3, List, ArrowUpDown, ArrowUp, FolderOpen, Users, MessageSquare } from "lucide-react"; import { toastWithSound as toast } from "@/lib/toast-with-sound"; import { getSettings } from "@/lib/settings"; import { openExternal } from "@/lib/utils"; -import { - GetAllAccountsFromDB, - GetAccountFromDB, - DeleteAccountFromDB, - SaveAccountToDB, - ExportAccountJSON, - ExportAccountsTXT, - UpdateAccountGroup, - GetAllGroups, - DownloadMediaWithMetadata, - StopDownload, - CheckFolderExists, - OpenFolder, - GetFolderPath, -} from "../../wailsjs/go/main/App"; +import { GetAllAccountsFromDB, GetAccountFromDB, DeleteAccountFromDB, SaveAccountToDB, ExportAccountJSON, ExportAccountsTXT, UpdateAccountGroup, GetAllGroups, DownloadMediaWithMetadata, StopDownload, CheckFolderExists, OpenFolder, GetFolderPath, } from "../../wailsjs/go/main/App"; import { EventsOn, EventsOff } from "../../wailsjs/runtime/runtime"; import { main } from "../../wailsjs/go/models"; - interface DownloadProgress { - current: number; - total: number; - percent: number; + current: number; + total: number; + percent: number; } - - - function formatNumberWithComma(num: number): string { - return num.toLocaleString(); + return num.toLocaleString(); } - function getRelativeTime(dateStr: string): string { - try { - const date = new Date(dateStr); - const now = new Date(); - const diffMs = now.getTime() - date.getTime(); - - const diffMinutes = Math.floor(diffMs / (1000 * 60)); - const diffHours = Math.floor(diffMs / (1000 * 60 * 60)); - const diffDays = Math.floor(diffMs / (1000 * 60 * 60 * 24)); - - if (diffDays > 0) { - const remainingHours = diffHours % 24; - return `(${diffDays}d ${remainingHours}h ago)`; - } else if (diffHours > 0) { - const remainingMinutes = diffMinutes % 60; - return `(${diffHours}h ${remainingMinutes}m ago)`; - } else if (diffMinutes > 0) { - return `(${diffMinutes}m ago)`; - } else { - return "(just now)"; + try { + const date = new Date(dateStr); + const now = new Date(); + const diffMs = now.getTime() - date.getTime(); + const diffMinutes = Math.floor(diffMs / (1000 * 60)); + const diffHours = Math.floor(diffMs / (1000 * 60 * 60)); + const diffDays = Math.floor(diffMs / (1000 * 60 * 60 * 24)); + if (diffDays > 0) { + const remainingHours = diffHours % 24; + return `(${diffDays}d ${remainingHours}h ago)`; + } + else if (diffHours > 0) { + const remainingMinutes = diffMinutes % 60; + return `(${diffHours}h ${remainingMinutes}m ago)`; + } + else if (diffMinutes > 0) { + return `(${diffMinutes}m ago)`; + } + else { + return "(just now)"; + } + } + catch { + return ""; } - } catch { - return ""; - } } - -// Using backend.AccountListItem from wailsjs/go/models import { backend } from "../../wailsjs/go/models"; type AccountListItem = backend.AccountListItem; - interface GroupInfo { - name: string; - color: string; + name: string; + color: string; } - interface DatabaseViewProps { - onBack: () => void; - onLoadAccount: (responseJSON: string, username: string) => void; - onUpdateSelected?: (usernames: string[]) => void; + onBack: () => void; + onLoadAccount: (responseJSON: string, username: string) => void; + onUpdateSelected?: (usernames: string[]) => void; } - const INITIAL_LOAD_COUNT = 12; const LOAD_MORE_COUNT = 12; - export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewProps) { - const [accounts, setAccounts] = useState([]); - const [loading, setLoading] = useState(true); - const [selectedIds, setSelectedIds] = useState>(new Set()); - const [visibleCount, setVisibleCount] = useState(INITIAL_LOAD_COUNT); - const [showScrollTop, setShowScrollTop] = useState(false); - const [groups, setGroups] = useState([]); - const [filterGroup, setFilterGroup] = useState("all"); - const [filterMediaType, setFilterMediaType] = useState("all"); - const [accountViewMode, setAccountViewMode] = useState<"public" | "private">("public"); - const [searchQuery, setSearchQuery] = useState(""); - const [sortOrder, setSortOrder] = useState<"newest" | "oldest" | "username-asc" | "username-desc" | "followers-high" | "followers-low" | "posts-high" | "posts-low" | "media-high" | "media-low">("newest"); - const [gridView, setGridView] = useState<"large" | "small" | "list">("list"); - const [editingAccount, setEditingAccount] = useState(null); - const [editGroupName, setEditGroupName] = useState(""); - const [editGroupColor, setEditGroupColor] = useState(""); - const [clearAllDialogOpen, setClearAllDialogOpen] = useState(false); - const [isDownloading, setIsDownloading] = useState(false); - const [downloadingAccountId, setDownloadingAccountId] = useState(null); - const [downloadProgress, setDownloadProgress] = useState(null); - const [isBulkDownloading, setIsBulkDownloading] = useState(false); - const [bulkDownloadCurrent, setBulkDownloadCurrent] = useState(0); - const [bulkDownloadTotal, setBulkDownloadTotal] = useState(0); - const stopBulkDownloadRef = useRef(false); - const loadMoreRef = useRef(null); - const [folderExistence, setFolderExistence] = useState>(new Map()); - - // Check folder existence for all accounts - const checkFolderExistence = async (accountsList: AccountListItem[]) => { - const settings = getSettings(); - const basePath = settings.downloadPath; - if (!basePath) return; - - const folderMap = new Map(); - - for (const account of accountsList) { - // For bookmarks/likes, check special folder names - let folderName = account.username; - if (account.username === "bookmarks") { - folderName = "My Bookmarks"; - } else if (account.username === "likes") { - folderName = "My Likes"; - } - - const exists = await CheckFolderExists(basePath, folderName); - folderMap.set(account.id, exists); - } - - setFolderExistence(folderMap); - }; - - const loadAccounts = async () => { - setLoading(true); - try { - const data = await GetAllAccountsFromDB(); - setAccounts(data || []); - const groupsData = await GetAllGroups(); - if (groupsData) { - setGroups(groupsData.map((g) => ({ name: g.name || "", color: g.color || "" }))); - } - // Check folder existence for all accounts - if (data && data.length > 0) { - checkFolderExistence(data); - } - } catch (error) { - console.error("Failed to load accounts:", error); - toast.error("Failed to load accounts"); - } finally { - setLoading(false); - } - }; - - useEffect(() => { - loadAccounts(); - }, []); - - // Reset visible count when grid view or filters change - useEffect(() => { - setVisibleCount(INITIAL_LOAD_COUNT); - }, [gridView, searchQuery, filterGroup, filterMediaType, accountViewMode, sortOrder]); - - // Listen for scroll to show/hide scroll-to-top button - // Listen for scroll to show/hide scroll-to-top button - useEffect(() => { - const handleScroll = () => { - setShowScrollTop(window.scrollY > 300); + const [accounts, setAccounts] = useState([]); + const [loading, setLoading] = useState(true); + const [selectedIds, setSelectedIds] = useState>(new Set()); + const [visibleCount, setVisibleCount] = useState(INITIAL_LOAD_COUNT); + const [showScrollTop, setShowScrollTop] = useState(false); + const [groups, setGroups] = useState([]); + const [filterGroup, setFilterGroup] = useState("all"); + const [filterMediaType, setFilterMediaType] = useState("all"); + const [accountViewMode, setAccountViewMode] = useState<"public" | "private">("public"); + const [searchQuery, setSearchQuery] = useState(""); + const [sortOrder, setSortOrder] = useState<"newest" | "oldest" | "username-asc" | "username-desc" | "followers-high" | "followers-low" | "posts-high" | "posts-low" | "media-high" | "media-low">("newest"); + const [gridView, setGridView] = useState<"large" | "small" | "list">("list"); + const [editingAccount, setEditingAccount] = useState(null); + const [editGroupName, setEditGroupName] = useState(""); + const [editGroupColor, setEditGroupColor] = useState(""); + const [clearAllDialogOpen, setClearAllDialogOpen] = useState(false); + const [isDownloading, setIsDownloading] = useState(false); + const [downloadingAccountId, setDownloadingAccountId] = useState(null); + const [downloadProgress, setDownloadProgress] = useState(null); + const [isBulkDownloading, setIsBulkDownloading] = useState(false); + const [bulkDownloadCurrent, setBulkDownloadCurrent] = useState(0); + const [bulkDownloadTotal, setBulkDownloadTotal] = useState(0); + const stopBulkDownloadRef = useRef(false); + const loadMoreRef = useRef(null); + const [folderExistence, setFolderExistence] = useState>(new Map()); + const checkFolderExistence = async (accountsList: AccountListItem[]) => { + const settings = getSettings(); + const basePath = settings.downloadPath; + if (!basePath) + return; + const folderMap = new Map(); + for (const account of accountsList) { + let folderName = account.username; + if (account.username === "bookmarks") { + folderName = "My Bookmarks"; + } + else if (account.username === "likes") { + folderName = "My Likes"; + } + const exists = await CheckFolderExists(basePath, folderName); + folderMap.set(account.id, exists); + } + setFolderExistence(folderMap); }; - window.addEventListener("scroll", handleScroll); - return () => window.removeEventListener("scroll", handleScroll); - }, []); - - // Intersection Observer for lazy loading - // Listen for download progress events - useEffect(() => { - const unsubscribe = EventsOn("download-progress", (progress: DownloadProgress) => { - setDownloadProgress(progress); - }); - return () => { - EventsOff("download-progress"); - unsubscribe(); - }; - }, []); - - const scrollToTop = () => { - window.scrollTo({ top: 0, behavior: "smooth" }); - }; - - // Separate accounts into public and private - const isPrivateAccount = (username: string) => - username === "bookmarks" || username === "likes"; - - const publicAccounts = accounts.filter((acc) => !isPrivateAccount(acc.username)); - const privateAccounts = accounts.filter((acc) => isPrivateAccount(acc.username)); - - const baseAccounts = accountViewMode === "public" ? publicAccounts : privateAccounts; - - const filteredAccounts = baseAccounts - .filter((acc) => { - // Filter by search query - if (searchQuery) { - const query = searchQuery.toLowerCase(); - const matchesUsername = acc.username.toLowerCase().includes(query); - const matchesName = acc.name.toLowerCase().includes(query); - if (!matchesUsername && !matchesName) return false; - } - // Filter by group - if (filterGroup !== "all") { - if (filterGroup === "ungrouped" && acc.group_name) return false; - if (filterGroup !== "ungrouped" && acc.group_name !== filterGroup) return false; - } - // Filter by media type - if (filterMediaType !== "all") { - const accMediaType = acc.media_type || "all"; - // "all-media" in filter matches "all" in database (fetched with All Media option) - if (filterMediaType === "all-media") { - if (accMediaType !== "all") return false; - } else { - if (accMediaType !== filterMediaType) return false; - } - } - return true; - }) - .sort((a, b) => { - switch (sortOrder) { - case "newest": { - const dateA = new Date(a.last_fetched).getTime(); - const dateB = new Date(b.last_fetched).getTime(); - return dateB - dateA; - } - case "oldest": { - const dateA = new Date(a.last_fetched).getTime(); - const dateB = new Date(b.last_fetched).getTime(); - return dateA - dateB; - } - case "username-asc": - return a.username.toLowerCase().localeCompare(b.username.toLowerCase()); - case "username-desc": - return b.username.toLowerCase().localeCompare(a.username.toLowerCase()); - case "followers-high": - return (b.followers_count || 0) - (a.followers_count || 0); - case "followers-low": - return (a.followers_count || 0) - (b.followers_count || 0); - case "posts-high": - return (b.statuses_count || 0) - (a.statuses_count || 0); - case "posts-low": - return (a.statuses_count || 0) - (b.statuses_count || 0); - case "media-high": - return (b.total_media || 0) - (a.total_media || 0); - case "media-low": - return (a.total_media || 0) - (b.total_media || 0); - default: - return 0; - } - }); - - // Intersection Observer for lazy loading - useEffect(() => { - const currentRef = loadMoreRef.current; - if (!currentRef) return; - - const observer = new IntersectionObserver( - (entries) => { - if (entries[0].isIntersecting) { - setVisibleCount((prev) => prev + LOAD_MORE_COUNT); - } - }, - { threshold: 0.1, rootMargin: "100px" } - ); - - observer.observe(currentRef); - - return () => { - observer.unobserve(currentRef); - }; - }, [filteredAccounts.length, visibleCount, loading]); - - const handleEditGroup = (account: AccountListItem) => { - setEditingAccount(account); - setEditGroupName(account.group_name || ""); - setEditGroupColor(account.group_color || "#3b82f6"); - }; - - const handleSaveGroup = async () => { - if (!editingAccount) return; - try { - await UpdateAccountGroup(editingAccount.id, editGroupName, editGroupColor); - toast.success(`Updated group for @${editingAccount.username}`); - setEditingAccount(null); - loadAccounts(); - } catch (error) { - toast.error("Failed to update group"); - } - }; - - const handleDelete = async (id: number, username: string) => { - try { - await DeleteAccountFromDB(id); - toast.success(`Deleted @${username}`); - loadAccounts(); - } catch (error) { - toast.error("Failed to delete account"); - } - }; - - const handleView = async (id: number, username: string) => { - try { - const responseJSON = await GetAccountFromDB(id); - onLoadAccount(responseJSON, username); - } catch (error) { - toast.error("Failed to load account data"); - } - }; - - const handleOpenFolder = async (username: string) => { - const settings = getSettings(); - let folderName = username; - if (username === "bookmarks") { - folderName = "My Bookmarks"; - } else if (username === "likes") { - folderName = "My Likes"; - } - const folderPath = await GetFolderPath(settings.downloadPath, folderName); - try { - await OpenFolder(folderPath); - } catch (error) { - toast.error("Failed to open folder"); - } - }; - - const handleDownload = async (id: number, username: string) => { - try { - const responseJSON = await GetAccountFromDB(id); - const data = JSON.parse(responseJSON); - const timeline = data.timeline || []; - - if (timeline.length === 0) { - toast.error("No media to download"); - return; - } - - const settings = getSettings(); - // Check if it's bookmarks or likes by checking account_info.nick or username (for backward compatibility) - const isBookmarks = data.account_info?.nick === "My Bookmarks" || username === "bookmarks"; - const isLikes = data.account_info?.nick === "My Likes" || username === "likes"; - let outputDir = settings.downloadPath; - if (isBookmarks) { - const separator = settings.downloadPath.includes("/") ? "/" : "\\"; - outputDir = `${settings.downloadPath}${separator}My Bookmarks`; - } else if (isLikes) { - const separator = settings.downloadPath.includes("/") ? "/" : "\\"; - outputDir = `${settings.downloadPath}${separator}My Likes`; - } - // Use actual username from account_info if available, otherwise use stored username - const actualUsername = data.account_info?.name || username; - - setIsDownloading(true); - setDownloadingAccountId(id); - setDownloadProgress({ current: 0, total: timeline.length, percent: 0 }); - - const request = new main.DownloadMediaWithMetadataRequest({ - items: timeline.map((item: { url: string; date: string; tweet_id: string; type: string; original_filename?: string; author_username?: string }) => new main.MediaItemRequest({ - url: item.url, - date: item.date, - tweet_id: item.tweet_id, - type: item.type, - original_filename: item.original_filename || "", - author_username: item.author_username || "", - })), - output_dir: outputDir, - username: actualUsername, - proxy: settings.proxy || "", - }); - - const response = await DownloadMediaWithMetadata(request); - - if (response.success) { - const parts: string[] = []; - if (response.downloaded > 0) { - parts.push(`${response.downloaded} file${response.downloaded !== 1 ? 's' : ''} downloaded`); - } - if (response.skipped > 0) { - parts.push(`${response.skipped} file${response.skipped !== 1 ? 's' : ''} already exist${response.skipped !== 1 ? '' : 's'}`); - } - if (response.failed > 0) { - parts.push(`${response.failed} failed`); - } - const message = parts.length > 0 ? `${parts.join(', ')} for @${username}` : `Download completed for @${username}`; - - // Use info toast if only skipped files (no downloaded, no failed) - if (response.downloaded === 0 && response.failed === 0 && response.skipped > 0) { - toast.info(message); - } else { - toast.success(message); - } - } else { - toast.error(response.message || "Download failed"); - } - } catch (error) { - const errorMsg = error instanceof Error ? error.message : String(error); - toast.error(`Download failed: ${errorMsg}`); - } finally { - setIsDownloading(false); - setDownloadingAccountId(null); - setDownloadProgress(null); - } - }; - - const handleStopDownload = async () => { - try { - const stopped = await StopDownload(); - if (stopped) { - toast.info("Download stopped"); - } - } catch (error) { - console.error("Failed to stop download:", error); - } - }; - - const handleBulkDownload = async () => { - const idsToDownload = Array.from(selectedIds); - if (idsToDownload.length === 0) { - toast.error("No accounts selected"); - return; - } - - setIsBulkDownloading(true); - setBulkDownloadTotal(idsToDownload.length); - setBulkDownloadCurrent(0); - stopBulkDownloadRef.current = false; - - const settings = getSettings(); - let totalDownloaded = 0; - let totalSkipped = 0; - let downloadedAccounts = 0; - - for (let i = 0; i < idsToDownload.length; i++) { - // Check stop flag using ref (works inside async loop) - if (stopBulkDownloadRef.current) { - toast.info("Bulk download stopped"); - break; - } - - const id = idsToDownload[i]; - const account = accounts.find((a) => a.id === id); - if (!account) continue; - - setBulkDownloadCurrent(i + 1); - setDownloadingAccountId(id); - - try { - const responseJSON = await GetAccountFromDB(id); - const data = JSON.parse(responseJSON); - const timeline = data.timeline || []; - - if (timeline.length === 0) continue; - - // Check again before starting download - if (stopBulkDownloadRef.current) { - toast.info("Bulk download stopped"); - break; - } - - setDownloadProgress({ current: 0, total: timeline.length, percent: 0 }); - - // Check if it's bookmarks or likes by checking account_info.nick or username (for backward compatibility) - const isBookmarks = data.account_info?.nick === "My Bookmarks" || account.username === "bookmarks"; - const isLikes = data.account_info?.nick === "My Likes" || account.username === "likes"; - let outputDir = settings.downloadPath; - if (isBookmarks) { - const separator = settings.downloadPath.includes("/") ? "/" : "\\"; - outputDir = `${settings.downloadPath}${separator}My Bookmarks`; - } else if (isLikes) { - const separator = settings.downloadPath.includes("/") ? "/" : "\\"; - outputDir = `${settings.downloadPath}${separator}My Likes`; - } - // Use actual username from account_info if available, otherwise use stored username - const actualUsername = data.account_info?.name || account.username; - - const request = new main.DownloadMediaWithMetadataRequest({ - items: timeline.map((item: { url: string; date: string; tweet_id: string; type: string; author_username?: string; original_filename?: string }) => new main.MediaItemRequest({ - url: item.url, - date: item.date, - tweet_id: item.tweet_id, - type: item.type, - author_username: item.author_username || "", - original_filename: item.original_filename || "", - })), - output_dir: outputDir, - username: actualUsername, - proxy: settings.proxy || "", - }); - - const response = await DownloadMediaWithMetadata(request); - if (response.success) { - totalDownloaded += response.downloaded; - totalSkipped += response.skipped || 0; - downloadedAccounts++; - } - } catch (error) { - console.error(`Failed to download @${account.username}:`, error); - } - } - - setIsBulkDownloading(false); - setDownloadingAccountId(null); - setDownloadProgress(null); - setBulkDownloadCurrent(0); - setBulkDownloadTotal(0); - - if ((totalDownloaded > 0 || totalSkipped > 0) && !stopBulkDownloadRef.current) { - const parts: string[] = []; - if (totalDownloaded > 0) { - parts.push(`${totalDownloaded} file${totalDownloaded !== 1 ? 's' : ''} downloaded`); - } - if (totalSkipped > 0) { - parts.push(`${totalSkipped} file${totalSkipped !== 1 ? 's' : ''} already exist${totalSkipped !== 1 ? '' : 's'}`); - } - const message = parts.length > 0 ? `${parts.join(', ')} from ${downloadedAccounts} account${downloadedAccounts !== 1 ? 's' : ''}` : `Download completed from ${downloadedAccounts} account${downloadedAccounts !== 1 ? 's' : ''}`; - - // Use info toast if only skipped files (no downloaded) - if (totalDownloaded === 0 && totalSkipped > 0) { - toast.info(message); - } else { - toast.success(message); - } - } - }; - - const handleStopBulkDownload = async () => { - stopBulkDownloadRef.current = true; - await handleStopDownload(); - }; - - const toggleSelect = (id: number) => { - setSelectedIds((prev) => { - const newSet = new Set(prev); - if (newSet.has(id)) { - newSet.delete(id); - } else { - newSet.add(id); - } - return newSet; - }); - }; - - const toggleSelectAll = () => { - if (selectedIds.size === filteredAccounts.length && filteredAccounts.length > 0) { - setSelectedIds(new Set()); - } else { - setSelectedIds(new Set(filteredAccounts.map((a) => a.id))); - } - }; - - const handleExportJSON = async () => { - const idsToExport = selectedIds.size > 0 ? Array.from(selectedIds) : accounts.map((a) => a.id); - - if (idsToExport.length === 0) { - toast.error("No accounts to export"); - return; - } - - const settings = getSettings(); - const outputDir = settings.downloadPath || ""; - - try { - let exported = 0; - for (const id of idsToExport) { - await ExportAccountJSON(id, outputDir); - exported++; - } - toast.success(`Exported ${exported} account(s) to ${outputDir}\\twitterxmediabatchdownloader_backups`); - } catch (error) { - toast.error("Failed to export"); - } - }; - - const handleExportTXT = async () => { - const idsToExport = selectedIds.size > 0 ? Array.from(selectedIds) : accounts.map((a) => a.id); - - if (idsToExport.length === 0) { - toast.error("No accounts to export"); - return; - } - - const settings = getSettings(); - const outputDir = settings.downloadPath || ""; - - try { - await ExportAccountsTXT(idsToExport, outputDir); - toast.success(`Exported ${formatNumberWithComma(idsToExport.length)} account(s) to ${outputDir}\\twitterxmediabatchdownloader_backups\\twitterxmediabatchdownloader_multiple.txt`); - } catch (error) { - toast.error("Failed to export"); - } - }; - - // Check if any selected accounts are private (bookmarks/likes) - const hasPrivateAccountSelected = () => { - if (selectedIds.size === 0) return false; - return Array.from(selectedIds).some((id) => { - const account = accounts.find((a) => a.id === id); - return account && isPrivateAccount(account.username); - }); - }; - - const handleImport = () => { - const input = document.createElement("input"); - input.type = "file"; - input.accept = ".json"; - input.multiple = true; - input.onchange = async (e) => { - const files = (e.target as HTMLInputElement).files; - if (!files || files.length === 0) return; - - let imported = 0; - for (const file of Array.from(files)) { + const loadAccounts = async () => { + setLoading(true); try { - const text = await file.text(); - const data = JSON.parse(text); - - // Detect media_type from imported file - let detectedMediaType = "all"; - - // Check if media_type is explicitly set in the file - if (data.media_type) { - detectedMediaType = data.media_type; - } - // If not, try to detect from media_list content - else if (data.media_list && Array.isArray(data.media_list)) { - const types = new Set(data.media_list.map((item: { media_type?: string; type?: string }) => - item.media_type || item.type - ).filter(Boolean)); - - // If all items are same type, use that type - if (types.size === 1) { - const singleType = Array.from(types)[0] as string; - if (singleType === "photo") detectedMediaType = "image"; - else if (singleType === "video") detectedMediaType = "video"; - else if (singleType === "animated_gif") detectedMediaType = "gif"; - else detectedMediaType = singleType; + const data = await GetAllAccountsFromDB(); + setAccounts(data || []); + const groupsData = await GetAllGroups(); + if (groupsData) { + setGroups(groupsData.map((g) => ({ name: g.name || "", color: g.color || "" }))); } - } - // Or detect from timeline content - else if (data.timeline && Array.isArray(data.timeline)) { - const types = new Set(data.timeline.map((item: { type?: string }) => item.type).filter(Boolean)); - - // If all items are same type, use that type - if (types.size === 1) { - const singleType = Array.from(types)[0] as string; - if (singleType === "photo") detectedMediaType = "image"; - else if (singleType === "video") detectedMediaType = "video"; - else if (singleType === "animated_gif") detectedMediaType = "gif"; - else if (singleType === "text") detectedMediaType = "text"; - else detectedMediaType = singleType; + if (data && data.length > 0) { + checkFolderExistence(data); } - } - - // Support new format (account_info + timeline) - if (data.account_info && data.timeline) { - await SaveAccountToDB( - data.account_info.name, // username/handle - data.account_info.nick, // display name - data.account_info.profile_image, - data.total_urls || data.timeline.length, - text, - detectedMediaType - ); - imported++; - } - // Support legacy format (username + media_list) - else if (data.username && data.media_list) { - // Convert legacy format to new format - const convertedData = { - account_info: { - name: data.username, - nick: data.nick || data.username, - date: "", - followers_count: data.followers || 0, - friends_count: data.following || 0, - profile_image: data.profile_image || "", - statuses_count: data.posts || 0, - }, - total_urls: data.media_list.length, - timeline: data.media_list.map((item: { url: string; date: string; tweet_id: string; type: string; media_type?: string }) => ({ - url: item.url, - date: item.date, - tweet_id: item.tweet_id, - type: item.media_type || item.type, - is_retweet: false, - })), - metadata: { - new_entries: data.media_list.length, - page: 0, - batch_size: 0, - has_more: false, - }, - }; - - await SaveAccountToDB( - convertedData.account_info.name, - convertedData.account_info.nick, - convertedData.account_info.profile_image, - convertedData.total_urls, - JSON.stringify(convertedData), - detectedMediaType - ); - imported++; - } - } catch (err) { - console.error(`Failed to import ${file.name}:`, err); } - } - - if (imported > 0) { - toast.success(`Imported ${imported} account(s)`); - loadAccounts(); - } else { - toast.error("No valid files imported"); - } + catch (error) { + console.error("Failed to load accounts:", error); + toast.error("Failed to load accounts"); + } + finally { + setLoading(false); + } }; - input.click(); - }; - - return ( -
+ useEffect(() => { + loadAccounts(); + }, []); + useEffect(() => { + setVisibleCount(INITIAL_LOAD_COUNT); + }, [gridView, searchQuery, filterGroup, filterMediaType, accountViewMode, sortOrder]); + useEffect(() => { + const handleScroll = () => { + setShowScrollTop(window.scrollY > 300); + }; + window.addEventListener("scroll", handleScroll); + return () => window.removeEventListener("scroll", handleScroll); + }, []); + useEffect(() => { + const unsubscribe = EventsOn("download-progress", (progress: DownloadProgress) => { + setDownloadProgress(progress); + }); + return () => { + EventsOff("download-progress"); + unsubscribe(); + }; + }, []); + const scrollToTop = () => { + window.scrollTo({ top: 0, behavior: "smooth" }); + }; + const isPrivateAccount = (username: string) => username === "bookmarks" || username === "likes"; + const publicAccounts = accounts.filter((acc) => !isPrivateAccount(acc.username)); + const privateAccounts = accounts.filter((acc) => isPrivateAccount(acc.username)); + const baseAccounts = accountViewMode === "public" ? publicAccounts : privateAccounts; + const filteredAccounts = baseAccounts + .filter((acc) => { + if (searchQuery) { + const query = searchQuery.toLowerCase(); + const matchesUsername = acc.username.toLowerCase().includes(query); + const matchesName = acc.name.toLowerCase().includes(query); + if (!matchesUsername && !matchesName) + return false; + } + if (filterGroup !== "all") { + if (filterGroup === "ungrouped" && acc.group_name) + return false; + if (filterGroup !== "ungrouped" && acc.group_name !== filterGroup) + return false; + } + if (filterMediaType !== "all") { + const accMediaType = acc.media_type || "all"; + if (filterMediaType === "all-media") { + if (accMediaType !== "all") + return false; + } + else { + if (accMediaType !== filterMediaType) + return false; + } + } + return true; + }) + .sort((a, b) => { + switch (sortOrder) { + case "newest": { + const dateA = new Date(a.last_fetched).getTime(); + const dateB = new Date(b.last_fetched).getTime(); + return dateB - dateA; + } + case "oldest": { + const dateA = new Date(a.last_fetched).getTime(); + const dateB = new Date(b.last_fetched).getTime(); + return dateA - dateB; + } + case "username-asc": + return a.username.toLowerCase().localeCompare(b.username.toLowerCase()); + case "username-desc": + return b.username.toLowerCase().localeCompare(a.username.toLowerCase()); + case "followers-high": + return (b.followers_count || 0) - (a.followers_count || 0); + case "followers-low": + return (a.followers_count || 0) - (b.followers_count || 0); + case "posts-high": + return (b.statuses_count || 0) - (a.statuses_count || 0); + case "posts-low": + return (a.statuses_count || 0) - (b.statuses_count || 0); + case "media-high": + return (b.total_media || 0) - (a.total_media || 0); + case "media-low": + return (a.total_media || 0) - (b.total_media || 0); + default: + return 0; + } + }); + useEffect(() => { + const currentRef = loadMoreRef.current; + if (!currentRef) + return; + const observer = new IntersectionObserver((entries) => { + if (entries[0].isIntersecting) { + setVisibleCount((prev) => prev + LOAD_MORE_COUNT); + } + }, { threshold: 0.1, rootMargin: "100px" }); + observer.observe(currentRef); + return () => { + observer.unobserve(currentRef); + }; + }, [filteredAccounts.length, visibleCount, loading]); + const handleEditGroup = (account: AccountListItem) => { + setEditingAccount(account); + setEditGroupName(account.group_name || ""); + setEditGroupColor(account.group_color || "#3b82f6"); + }; + const handleSaveGroup = async () => { + if (!editingAccount) + return; + try { + await UpdateAccountGroup(editingAccount.id, editGroupName, editGroupColor); + toast.success(`Updated group for @${editingAccount.username}`); + setEditingAccount(null); + loadAccounts(); + } + catch (error) { + toast.error("Failed to update group"); + } + }; + const handleDelete = async (id: number, username: string) => { + try { + await DeleteAccountFromDB(id); + toast.success(`Deleted @${username}`); + loadAccounts(); + } + catch (error) { + toast.error("Failed to delete account"); + } + }; + const handleView = async (id: number, username: string) => { + try { + const responseJSON = await GetAccountFromDB(id); + onLoadAccount(responseJSON, username); + } + catch (error) { + toast.error("Failed to load account data"); + } + }; + const handleOpenFolder = async (username: string) => { + const settings = getSettings(); + let folderName = username; + if (username === "bookmarks") { + folderName = "My Bookmarks"; + } + else if (username === "likes") { + folderName = "My Likes"; + } + const folderPath = await GetFolderPath(settings.downloadPath, folderName); + try { + await OpenFolder(folderPath); + } + catch (error) { + toast.error("Failed to open folder"); + } + }; + const handleDownload = async (id: number, username: string) => { + try { + const responseJSON = await GetAccountFromDB(id); + const data = JSON.parse(responseJSON); + const timeline = data.timeline || []; + if (timeline.length === 0) { + toast.error("No media to download"); + return; + } + const settings = getSettings(); + const isBookmarks = data.account_info?.nick === "My Bookmarks" || username === "bookmarks"; + const isLikes = data.account_info?.nick === "My Likes" || username === "likes"; + let outputDir = settings.downloadPath; + if (isBookmarks) { + const separator = settings.downloadPath.includes("/") ? "/" : "\\"; + outputDir = `${settings.downloadPath}${separator}My Bookmarks`; + } + else if (isLikes) { + const separator = settings.downloadPath.includes("/") ? "/" : "\\"; + outputDir = `${settings.downloadPath}${separator}My Likes`; + } + const actualUsername = data.account_info?.name || username; + setIsDownloading(true); + setDownloadingAccountId(id); + setDownloadProgress({ current: 0, total: timeline.length, percent: 0 }); + const request = new main.DownloadMediaWithMetadataRequest({ + items: timeline.map((item: { + url: string; + date: string; + tweet_id: string; + type: string; + original_filename?: string; + author_username?: string; + }) => new main.MediaItemRequest({ + url: item.url, + date: item.date, + tweet_id: item.tweet_id, + type: item.type, + original_filename: item.original_filename || "", + author_username: item.author_username || "", + })), + output_dir: outputDir, + username: actualUsername, + proxy: settings.proxy || "", + }); + const response = await DownloadMediaWithMetadata(request); + if (response.success) { + const parts: string[] = []; + if (response.downloaded > 0) { + parts.push(`${response.downloaded} file${response.downloaded !== 1 ? 's' : ''} downloaded`); + } + if (response.skipped > 0) { + parts.push(`${response.skipped} file${response.skipped !== 1 ? 's' : ''} already exist${response.skipped !== 1 ? '' : 's'}`); + } + if (response.failed > 0) { + parts.push(`${response.failed} failed`); + } + const message = parts.length > 0 ? `${parts.join(', ')} for @${username}` : `Download completed for @${username}`; + if (response.downloaded === 0 && response.failed === 0 && response.skipped > 0) { + toast.info(message); + } + else { + toast.success(message); + } + } + else { + toast.error(response.message || "Download failed"); + } + } + catch (error) { + const errorMsg = error instanceof Error ? error.message : String(error); + toast.error(`Download failed: ${errorMsg}`); + } + finally { + setIsDownloading(false); + setDownloadingAccountId(null); + setDownloadProgress(null); + } + }; + const handleStopDownload = async () => { + try { + const stopped = await StopDownload(); + if (stopped) { + toast.info("Download stopped"); + } + } + catch (error) { + console.error("Failed to stop download:", error); + } + }; + const handleBulkDownload = async () => { + const idsToDownload = Array.from(selectedIds); + if (idsToDownload.length === 0) { + toast.error("No accounts selected"); + return; + } + setIsBulkDownloading(true); + setBulkDownloadTotal(idsToDownload.length); + setBulkDownloadCurrent(0); + stopBulkDownloadRef.current = false; + const settings = getSettings(); + let totalDownloaded = 0; + let totalSkipped = 0; + let downloadedAccounts = 0; + for (let i = 0; i < idsToDownload.length; i++) { + if (stopBulkDownloadRef.current) { + toast.info("Bulk download stopped"); + break; + } + const id = idsToDownload[i]; + const account = accounts.find((a) => a.id === id); + if (!account) + continue; + setBulkDownloadCurrent(i + 1); + setDownloadingAccountId(id); + try { + const responseJSON = await GetAccountFromDB(id); + const data = JSON.parse(responseJSON); + const timeline = data.timeline || []; + if (timeline.length === 0) + continue; + if (stopBulkDownloadRef.current) { + toast.info("Bulk download stopped"); + break; + } + setDownloadProgress({ current: 0, total: timeline.length, percent: 0 }); + const isBookmarks = data.account_info?.nick === "My Bookmarks" || account.username === "bookmarks"; + const isLikes = data.account_info?.nick === "My Likes" || account.username === "likes"; + let outputDir = settings.downloadPath; + if (isBookmarks) { + const separator = settings.downloadPath.includes("/") ? "/" : "\\"; + outputDir = `${settings.downloadPath}${separator}My Bookmarks`; + } + else if (isLikes) { + const separator = settings.downloadPath.includes("/") ? "/" : "\\"; + outputDir = `${settings.downloadPath}${separator}My Likes`; + } + const actualUsername = data.account_info?.name || account.username; + const request = new main.DownloadMediaWithMetadataRequest({ + items: timeline.map((item: { + url: string; + date: string; + tweet_id: string; + type: string; + author_username?: string; + original_filename?: string; + }) => new main.MediaItemRequest({ + url: item.url, + date: item.date, + tweet_id: item.tweet_id, + type: item.type, + author_username: item.author_username || "", + original_filename: item.original_filename || "", + })), + output_dir: outputDir, + username: actualUsername, + proxy: settings.proxy || "", + }); + const response = await DownloadMediaWithMetadata(request); + if (response.success) { + totalDownloaded += response.downloaded; + totalSkipped += response.skipped || 0; + downloadedAccounts++; + } + } + catch (error) { + console.error(`Failed to download @${account.username}:`, error); + } + } + setIsBulkDownloading(false); + setDownloadingAccountId(null); + setDownloadProgress(null); + setBulkDownloadCurrent(0); + setBulkDownloadTotal(0); + if ((totalDownloaded > 0 || totalSkipped > 0) && !stopBulkDownloadRef.current) { + const parts: string[] = []; + if (totalDownloaded > 0) { + parts.push(`${totalDownloaded} file${totalDownloaded !== 1 ? 's' : ''} downloaded`); + } + if (totalSkipped > 0) { + parts.push(`${totalSkipped} file${totalSkipped !== 1 ? 's' : ''} already exist${totalSkipped !== 1 ? '' : 's'}`); + } + const message = parts.length > 0 ? `${parts.join(', ')} from ${downloadedAccounts} account${downloadedAccounts !== 1 ? 's' : ''}` : `Download completed from ${downloadedAccounts} account${downloadedAccounts !== 1 ? 's' : ''}`; + if (totalDownloaded === 0 && totalSkipped > 0) { + toast.info(message); + } + else { + toast.success(message); + } + } + }; + const handleStopBulkDownload = async () => { + stopBulkDownloadRef.current = true; + await handleStopDownload(); + }; + const toggleSelect = (id: number) => { + setSelectedIds((prev) => { + const newSet = new Set(prev); + if (newSet.has(id)) { + newSet.delete(id); + } + else { + newSet.add(id); + } + return newSet; + }); + }; + const toggleSelectAll = () => { + if (selectedIds.size === filteredAccounts.length && filteredAccounts.length > 0) { + setSelectedIds(new Set()); + } + else { + setSelectedIds(new Set(filteredAccounts.map((a) => a.id))); + } + }; + const handleExportJSON = async () => { + const idsToExport = selectedIds.size > 0 ? Array.from(selectedIds) : accounts.map((a) => a.id); + if (idsToExport.length === 0) { + toast.error("No accounts to export"); + return; + } + const settings = getSettings(); + const outputDir = settings.downloadPath || ""; + try { + let exported = 0; + for (const id of idsToExport) { + await ExportAccountJSON(id, outputDir); + exported++; + } + toast.success(`Exported ${exported} account(s) to ${outputDir}\\twitterxmediabatchdownloader_backups`); + } + catch (error) { + toast.error("Failed to export"); + } + }; + const handleExportTXT = async () => { + const idsToExport = selectedIds.size > 0 ? Array.from(selectedIds) : accounts.map((a) => a.id); + if (idsToExport.length === 0) { + toast.error("No accounts to export"); + return; + } + const settings = getSettings(); + const outputDir = settings.downloadPath || ""; + try { + await ExportAccountsTXT(idsToExport, outputDir); + toast.success(`Exported ${formatNumberWithComma(idsToExport.length)} account(s) to ${outputDir}\\twitterxmediabatchdownloader_backups\\twitterxmediabatchdownloader_multiple.txt`); + } + catch (error) { + toast.error("Failed to export"); + } + }; + const hasPrivateAccountSelected = () => { + if (selectedIds.size === 0) + return false; + return Array.from(selectedIds).some((id) => { + const account = accounts.find((a) => a.id === id); + return account && isPrivateAccount(account.username); + }); + }; + const handleImport = () => { + const input = document.createElement("input"); + input.type = "file"; + input.accept = ".json"; + input.multiple = true; + input.onchange = async (e) => { + const files = (e.target as HTMLInputElement).files; + if (!files || files.length === 0) + return; + let imported = 0; + for (const file of Array.from(files)) { + try { + const text = await file.text(); + const data = JSON.parse(text); + let detectedMediaType = "all"; + if (data.media_type) { + detectedMediaType = data.media_type; + } + else if (data.media_list && Array.isArray(data.media_list)) { + const types = new Set(data.media_list.map((item: { + media_type?: string; + type?: string; + }) => item.media_type || item.type).filter(Boolean)); + if (types.size === 1) { + const singleType = Array.from(types)[0] as string; + if (singleType === "photo") + detectedMediaType = "image"; + else if (singleType === "video") + detectedMediaType = "video"; + else if (singleType === "animated_gif") + detectedMediaType = "gif"; + else + detectedMediaType = singleType; + } + } + else if (data.timeline && Array.isArray(data.timeline)) { + const types = new Set(data.timeline.map((item: { + type?: string; + }) => item.type).filter(Boolean)); + if (types.size === 1) { + const singleType = Array.from(types)[0] as string; + if (singleType === "photo") + detectedMediaType = "image"; + else if (singleType === "video") + detectedMediaType = "video"; + else if (singleType === "animated_gif") + detectedMediaType = "gif"; + else if (singleType === "text") + detectedMediaType = "text"; + else + detectedMediaType = singleType; + } + } + if (data.account_info && data.timeline) { + await SaveAccountToDB(data.account_info.name, data.account_info.nick, data.account_info.profile_image, data.total_urls || data.timeline.length, text, detectedMediaType); + imported++; + } + else if (data.username && data.media_list) { + const convertedData = { + account_info: { + name: data.username, + nick: data.nick || data.username, + date: "", + followers_count: data.followers || 0, + friends_count: data.following || 0, + profile_image: data.profile_image || "", + statuses_count: data.posts || 0, + }, + total_urls: data.media_list.length, + timeline: data.media_list.map((item: { + url: string; + date: string; + tweet_id: string; + type: string; + media_type?: string; + }) => ({ + url: item.url, + date: item.date, + tweet_id: item.tweet_id, + type: item.media_type || item.type, + is_retweet: false, + })), + metadata: { + new_entries: data.media_list.length, + page: 0, + batch_size: 0, + has_more: false, + }, + }; + await SaveAccountToDB(convertedData.account_info.name, convertedData.account_info.nick, convertedData.account_info.profile_image, convertedData.total_urls, JSON.stringify(convertedData), detectedMediaType); + imported++; + } + } + catch (err) { + console.error(`Failed to import ${file.name}:`, err); + } + } + if (imported > 0) { + toast.success(`Imported ${imported} account(s)`); + loadAccounts(); + } + else { + toast.error("No valid files imported"); + } + }; + input.click(); + }; + return (

Saved Accounts

- {/* Public/Private Toggle */} +
- -
@@ -791,7 +679,7 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr Import JSON @@ -801,7 +689,7 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr @@ -809,75 +697,64 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr - + Export JSON - + Export TXT - Update Selected ({formatNumberWithComma(selectedIds.size)}) - {isBulkDownloading ? ( - + {isBulkDownloading ? ( Stop Bulk Download - - ) : ( - + ) : ( Download Selected ({formatNumberWithComma(selectedIds.size)}) - - )} + )} @@ -885,13 +762,8 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr
-
@@ -901,23 +773,21 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr - @@ -926,61 +796,39 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr
- {loading ? ( -
Loading...
- ) : accounts.length === 0 ? ( -
+ {loading ? (
Loading...
) : accounts.length === 0 ? (
No saved accounts yet. Fetch a user's media to save it here. -
- ) : ( -
- {/* Row 1: Select All + Search Bar (Search only for public) */} +
) : (
+
- 0} - onCheckedChange={toggleSelectAll} - /> + 0} onCheckedChange={toggleSelectAll}/> Select all {selectedIds.size > 0 && `(${formatNumberWithComma(selectedIds.size)} selected)`}
- {/* Search Bar - Only show for public accounts */} - {accountViewMode === "public" && ( -
- - { + + {accountViewMode === "public" && (
+ + { setSearchQuery(e.target.value); - }} - className="pl-9 h-9" - /> - {searchQuery && ( - - )} -
- )} + }} className="pl-9 h-9"/> + {searchQuery && ()} +
)}
- {/* Row 2: Sort, Filters, View Toggle */} +
- {/* Sort Order - only for public accounts */} - {accountViewMode === "public" && ( - setSortOrder(v as typeof sortOrder)}> - - + + Newest @@ -994,15 +842,13 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr Media Count (High-Low) Media Count (Low-High) - - )} + )} - {/* Media Type Filter - only for public accounts */} - {accountViewMode === "public" && ( - - - + + @@ -1010,586 +856,331 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr - + All Media - + Images - - + GIFs - + Text - - )} + )} - {/* Group Filter - only for public accounts */} - {accountViewMode === "public" && ( - - - + + All Groups Ungrouped - {groups.map((group) => ( - + {groups.map((group) => ( - + {group.name} - - ))} + ))} - - )} + )} -
+
- {/* Grid View Toggle - only for public accounts */} - {accountViewMode === "public" && ( -
- - - -
- )} +
)}
- {/* Bulk Download Progress */} - {isBulkDownloading && ( -
+ + {isBulkDownloading && (
Downloading account {bulkDownloadCurrent} of {bulkDownloadTotal} {Math.round((bulkDownloadCurrent / bulkDownloadTotal) * 100)}%
- -
- )} + +
)} - {/* Grid View: Large */} - {gridView === "large" && ( -
+ + {gridView === "large" && (
{filteredAccounts - .slice(0, visibleCount) - .map((account) => ( -
- {/* Checkbox - Top Left */} - toggleSelect(account.id)} - className="absolute top-2 left-2 z-10" - /> - {/* Media Type Badge - Top Right */} + .slice(0, visibleCount) + .map((account) => (
+ + toggleSelect(account.id)} className="absolute top-2 left-2 z-10"/> +
- - {account.media_type === "text" ? : - account.media_type === "image" ? : - account.media_type === "video" ?
- {isPrivateAccount(account.username) ? ( -
handleView(account.id, account.username)} - > - {account.username === "bookmarks" ? ( - - ) : ( - - )} -
- ) : ( - {account.name} handleView(account.id, account.username)} - /> - )} + {isPrivateAccount(account.username) ? (
handleView(account.id, account.username)}> + {account.username === "bookmarks" ? () : ()} +
) : ({account.name} handleView(account.id, account.username)}/>)}
{account.name}
- {!isPrivateAccount(account.username) && ( - - )} + )}
{formatNumberWithComma(account.total_media)} media
- {downloadingAccountId === account.id ? ( - - ) : ( - - )} - ) : ()} + - {!isPrivateAccount(account.username) && ( - handleEditGroup(account)}> - + {!isPrivateAccount(account.username) && ( handleEditGroup(account)}> + Edit Group - - )} - { - const settings = getSettings(); - const outputDir = settings.downloadPath || ""; - try { - await ExportAccountJSON(account.id, outputDir); - toast.success(`Exported @${account.username}`); - } catch (error) { - toast.error("Failed to export"); - } - }} - > - + )} + { + const settings = getSettings(); + const outputDir = settings.downloadPath || ""; + try { + await ExportAccountJSON(account.id, outputDir); + toast.success(`Exported @${account.username}`); + } + catch (error) { + toast.error("Failed to export"); + } + }}> + Export JSON - handleDelete(account.id, account.username)} - className="text-destructive focus:text-destructive" - > - + handleDelete(account.id, account.username)} className="text-destructive focus:text-destructive"> + Delete
- {/* Progress bar */} - {downloadingAccountId === account.id && downloadProgress && ( -
+ + {downloadingAccountId === account.id && downloadProgress && (
{downloadProgress.current}/{downloadProgress.total} {downloadProgress.percent}%
- -
- )} -
- ))} -
- )} + +
)} +
))} +
)} - {/* Grid View: Small */} - {gridView === "small" && ( -
+ + {gridView === "small" && (
{filteredAccounts - .slice(0, visibleCount) - .map((account) => ( -
- {/* Checkbox - Top Left */} - toggleSelect(account.id)} - className="absolute top-1.5 left-1.5 z-10" - /> - {/* Media Type Badge & Incomplete - Top Right */} + .slice(0, visibleCount) + .map((account) => (
+ + toggleSelect(account.id)} className="absolute top-1.5 left-1.5 z-10"/> +
- - {account.media_type === "text" ? : - account.media_type === "image" ? : - account.media_type === "video" ?
- {isPrivateAccount(account.username) ? ( -
handleView(account.id, account.username)} - > - {account.username === "bookmarks" ? ( - - ) : ( - - )} -
- ) : ( - {account.name} handleView(account.id, account.username)} - /> - )} + {isPrivateAccount(account.username) ? (
handleView(account.id, account.username)}> + {account.username === "bookmarks" ? () : ()} +
) : ({account.name} handleView(account.id, account.username)}/>)}
{account.name}
- {!isPrivateAccount(account.username) && ( - - )} + )}
{formatNumberWithComma(account.total_media)} media
- {downloadingAccountId === account.id ? ( - - ) : ( - - )} - ) : ()} + - {!isPrivateAccount(account.username) && ( - handleEditGroup(account)}> - + {!isPrivateAccount(account.username) && ( handleEditGroup(account)}> + Edit Group - - )} - { - const settings = getSettings(); - const outputDir = settings.downloadPath || ""; - try { - await ExportAccountJSON(account.id, outputDir); - toast.success(`Exported @${account.username}`); - } catch (error) { - toast.error("Failed to export"); - } - }} - > - + )} + { + const settings = getSettings(); + const outputDir = settings.downloadPath || ""; + try { + await ExportAccountJSON(account.id, outputDir); + toast.success(`Exported @${account.username}`); + } + catch (error) { + toast.error("Failed to export"); + } + }}> + Export JSON - handleDelete(account.id, account.username)} - className="text-destructive focus:text-destructive" - > - + handleDelete(account.id, account.username)} className="text-destructive focus:text-destructive"> + Delete
- {/* Progress bar */} - {downloadingAccountId === account.id && downloadProgress && ( -
+ + {downloadingAccountId === account.id && downloadProgress && (
{downloadProgress.current}/{downloadProgress.total} {downloadProgress.percent}%
- -
- )} -
- ))} -
- )} + +
)} +
))} +
)} - {/* List View */} + {gridView === "list" && filteredAccounts - .slice(0, visibleCount) - .map((account, index) => ( -
+ .slice(0, visibleCount) + .map((account, index) => (
- toggleSelect(account.id)} - /> + toggleSelect(account.id)}/> {index + 1} - {isPrivateAccount(account.username) ? ( -
handleView(account.id, account.username)} - > - {account.username === "bookmarks" ? ( - - ) : ( - - )} -
- ) : ( - {account.name} handleView(account.id, account.username)} - /> - )} + {isPrivateAccount(account.username) ? (
handleView(account.id, account.username)}> + {account.username === "bookmarks" ? () : ()} +
) : ({account.name} handleView(account.id, account.username)}/>)}
{account.name} - {/* Stats Badges: Followers, Posts, Media */} - {!isPrivateAccount(account.username) && (account.followers_count > 0 || account.statuses_count > 0) && ( - <> + + {!isPrivateAccount(account.username) && (account.followers_count > 0 || account.statuses_count > 0) && (<> - + {formatNumberWithComma(account.followers_count)} - + {formatNumberWithComma(account.statuses_count)} - - )} + )} - + {formatNumberWithComma(account.total_media)} - {/* Media Type Badge */} - - {account.media_type === "text" ? <> Text : - account.media_type === "image" ? <> Images : - account.media_type === "video" ? <>
- {!isPrivateAccount(account.username) && ( - - )} + )}
{account.last_fetched} {getRelativeTime(account.last_fetched)}
- {downloadingAccountId === account.id ? ( - + {downloadingAccountId === account.id ? ( - Stop Download - - ) : ( - + ) : ( - Download All Media - - )} + )} - Open Folder @@ -1599,43 +1190,37 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr More Options - {!isPrivateAccount(account.username) && ( - handleEditGroup(account)}> - + {!isPrivateAccount(account.username) && ( handleEditGroup(account)}> + Edit Group - - )} - { - const settings = getSettings(); - const outputDir = settings.downloadPath || ""; - try { - await ExportAccountJSON(account.id, outputDir); - toast.success(`Exported @${account.username} to ${outputDir}\\twitterxmediabatchdownloader_backups`); - } catch (error) { - toast.error("Failed to export"); - } - }} - > - + )} + { + const settings = getSettings(); + const outputDir = settings.downloadPath || ""; + try { + await ExportAccountJSON(account.id, outputDir); + toast.success(`Exported @${account.username} to ${outputDir}\\twitterxmediabatchdownloader_backups`); + } + catch (error) { + toast.error("Failed to export"); + } + }}> + Export JSON - { - e.preventDefault(); - }} - className="text-destructive focus:text-destructive" - > - + { + e.preventDefault(); + }} className="text-destructive focus:text-destructive"> + Delete @@ -1643,7 +1228,7 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr
@@ -1654,10 +1239,7 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr - @@ -1667,54 +1249,35 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr
- {/* Progress bar for this account */} - {downloadingAccountId === account.id && downloadProgress && ( -
+ + {downloadingAccountId === account.id && downloadProgress && (
Downloading {downloadProgress.current} of {downloadProgress.total} {downloadProgress.percent}%
- -
- )} -
- ))} + +
)} +
))} - {/* Load More Trigger (invisible) */} -
= filteredAccounts.length ? 'hidden' : ''}`} - /> + +
= filteredAccounts.length ? 'hidden' : ''}`}/> -
- )} +
)} - {/* Scroll to Top Button */} - {showScrollTop && ( - - )} + + {showScrollTop && ()} - {/* Edit Group Dialog */} + !open && setEditingAccount(null)}>
-
@@ -1727,92 +1290,53 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr
- setEditGroupName(e.target.value)} - className="flex-1" - /> - {groups.length > 0 && ( - setEditGroupName(e.target.value)} className="flex-1"/> + {groups.length > 0 && ( - )} - {editGroupName && ( - + )} + {editGroupName && ( - Remove from Group - - )} + )}
- setEditGroupColor(e.target.value)} - className="absolute inset-0 w-full h-full opacity-0 cursor-pointer" - /> -
+ setEditGroupColor(e.target.value)} className="absolute inset-0 w-full h-full opacity-0 cursor-pointer"/> +
- setEditGroupColor(e.target.value)} - placeholder="#3b82f6" - className="w-28 font-mono text-sm" - /> + setEditGroupColor(e.target.value)} placeholder="#3b82f6" className="w-28 font-mono text-sm"/> - Random Color @@ -1830,6 +1354,5 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr
-
- ); +
); } diff --git a/frontend/src/components/DebugLoggerPage.tsx b/frontend/src/components/DebugLoggerPage.tsx index e33cbcd..f2d8853 100644 --- a/frontend/src/components/DebugLoggerPage.tsx +++ b/frontend/src/components/DebugLoggerPage.tsx @@ -2,99 +2,72 @@ import { useState, useEffect, useRef } from "react"; import { Trash2, Copy, Check } from "lucide-react"; import { Button } from "@/components/ui/button"; import { logger, type LogEntry } from "@/lib/logger"; - const levelColors: Record = { - info: "text-blue-500", - success: "text-green-500", - warning: "text-yellow-500", - error: "text-red-500", - debug: "text-gray-500", + info: "text-blue-500", + success: "text-green-500", + warning: "text-yellow-500", + error: "text-red-500", + debug: "text-gray-500", }; - function formatTime(date: Date): string { - return date.toLocaleTimeString("en-US", { - hour12: false, - hour: "2-digit", - minute: "2-digit", - second: "2-digit", - }); -} - -export function DebugLoggerPage() { - const [logs, setLogs] = useState([]); - const [copied, setCopied] = useState(false); - const scrollRef = useRef(null); - - useEffect(() => { - const unsubscribe = logger.subscribe(() => { - setLogs(logger.getLogs()); + return date.toLocaleTimeString("en-US", { + hour12: false, + hour: "2-digit", + minute: "2-digit", + second: "2-digit", }); - setLogs(logger.getLogs()); - return () => { - unsubscribe(); +} +export function DebugLoggerPage() { + const [logs, setLogs] = useState([]); + const [copied, setCopied] = useState(false); + const scrollRef = useRef(null); + useEffect(() => { + const unsubscribe = logger.subscribe(() => { + setLogs(logger.getLogs()); + }); + setLogs(logger.getLogs()); + return () => { + unsubscribe(); + }; + }, []); + useEffect(() => { + if (scrollRef.current) { + scrollRef.current.scrollTop = scrollRef.current.scrollHeight; + } + }, [logs]); + const handleClear = () => { + logger.clear(); }; - }, []); - - useEffect(() => { - if (scrollRef.current) { - scrollRef.current.scrollTop = scrollRef.current.scrollHeight; - } - }, [logs]); - - const handleClear = () => { - logger.clear(); - }; - - const handleCopy = async () => { - const logText = logs - .map((log) => `[${formatTime(log.timestamp)}] [${log.level}] ${log.message}`) - .join("\n"); - - try { - await navigator.clipboard.writeText(logText); - setCopied(true); - setTimeout(() => setCopied(false), 500); - } catch (err) { - console.error("Failed to copy logs:", err); - } - }; - - return ( -
+ const handleCopy = async () => { + const logText = logs + .map((log) => `[${formatTime(log.timestamp)}] [${log.level}] ${log.message}`) + .join("\n"); + try { + await navigator.clipboard.writeText(logText); + setCopied(true); + setTimeout(() => setCopied(false), 500); + } + catch (err) { + console.error("Failed to copy logs:", err); + } + }; + return (

Debug Logs

- -
-
- {logs.length === 0 ? ( -

no logs yet...

- ) : ( - logs.map((log, i) => ( -
+
+ {logs.length === 0 ? (

no logs yet...

) : (logs.map((log, i) => (
[{formatTime(log.timestamp)}] @@ -102,10 +75,7 @@ export function DebugLoggerPage() { [{log.level}] {log.message} -
- )) - )} +
)))}
-
- ); +
); } diff --git a/frontend/src/components/DependencySetupDialog.tsx b/frontend/src/components/DependencySetupDialog.tsx new file mode 100644 index 0000000..589298a --- /dev/null +++ b/frontend/src/components/DependencySetupDialog.tsx @@ -0,0 +1,103 @@ +import { useEffect, useState } from "react"; +import { AlertCircle, Download, Package } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle, } from "@/components/ui/dialog"; +import { Spinner } from "@/components/ui/spinner"; +import { toastWithSound as toast } from "@/lib/toast-with-sound"; +import { DownloadExtractor, IsExtractorInstalled } from "../../wailsjs/go/main/App"; +interface DependencySetupDialogProps { + onInstalled?: () => void; +} +export function DependencySetupDialog({ onInstalled }: DependencySetupDialogProps) { + const [open, setOpen] = useState(false); + const [checking, setChecking] = useState(true); + const [downloading, setDownloading] = useState(false); + const [errorMessage, setErrorMessage] = useState(null); + useEffect(() => { + let cancelled = false; + const checkDependency = async () => { + try { + const installed = await IsExtractorInstalled(); + if (!cancelled) { + setOpen(!installed); + setErrorMessage(null); + } + } + catch { + if (!cancelled) { + setOpen(true); + setErrorMessage("Couldn't verify the core dependency. You can install it now."); + } + } + finally { + if (!cancelled) { + setChecking(false); + } + } + }; + checkDependency(); + return () => { + cancelled = true; + }; + }, []); + const handleDownload = async () => { + setDownloading(true); + setErrorMessage(null); + try { + await DownloadExtractor(); + setOpen(false); + toast.success("Core dependency installed"); + onInstalled?.(); + } + catch (error) { + const message = error instanceof Error ? error.message : String(error); + setErrorMessage(message || "Failed to download dependency."); + toast.error("Failed to download dependency"); + } + finally { + setDownloading(false); + } + }; + if (checking && !open) { + return null; + } + return ( !downloading && setOpen(nextOpen)}> + + +
+ +
+ Download Core Dependency + + This app now uses a prebuilt xtractor binary from GitHub releases. Download it once so fetching can work on this device. + +
+ +
+
+ The binary is stored in the app data folder and reused on the next launch. Optional tools like FFmpeg and ExifTool stay available in Settings. +
+ + {errorMessage && (
+ +

{errorMessage}

+
)} +
+ + + + + +
+
); +} diff --git a/frontend/src/components/FetchHistory.tsx b/frontend/src/components/FetchHistory.tsx index bed48f2..f4b819f 100644 --- a/frontend/src/components/FetchHistory.tsx +++ b/frontend/src/components/FetchHistory.tsx @@ -1,58 +1,34 @@ import { Button } from "@/components/ui/button"; import { X, User } from "lucide-react"; - export interface HistoryItem { - id: string; - username: string; - name: string; - image: string; - mediaCount: number; - timestamp: number; + id: string; + username: string; + name: string; + image: string; + mediaCount: number; + timestamp: number; } - interface FetchHistoryProps { - history: HistoryItem[]; - onSelect: (item: HistoryItem) => void; - onRemove: (id: string) => void; + history: HistoryItem[]; + onSelect: (item: HistoryItem) => void; + onRemove: (id: string) => void; } - export function FetchHistory({ history, onSelect, onRemove }: FetchHistoryProps) { - if (history.length === 0) return null; - - return ( -
+ if (history.length === 0) + return null; + return (

Recent Fetches

- {history.map((item) => ( -
onSelect(item)} - > - {item.image ? ( - {item.name} - ) : ( - - )} + {history.map((item) => (
onSelect(item)}> + {item.image ? ({item.name}) : ()} @{item.username} - -
- ))} +
))}
-
- ); +
); } diff --git a/frontend/src/components/Header.tsx b/frontend/src/components/Header.tsx index 81f113e..3a928e7 100644 --- a/frontend/src/components/Header.tsx +++ b/frontend/src/components/Header.tsx @@ -1,63 +1,39 @@ import { Badge } from "@/components/ui/badge"; -import { - Tooltip, - TooltipContent, - TooltipTrigger, -} from "@/components/ui/tooltip"; +import { Tooltip, TooltipContent, TooltipTrigger, } from "@/components/ui/tooltip"; import { openExternal } from "@/lib/utils"; import { formatRelativeTime } from "@/lib/relative-time"; - interface HeaderProps { - version: string; - hasUpdate: boolean; - releaseDate?: string | null; + version: string; + hasUpdate: boolean; + releaseDate?: string | null; } - export function Header({ version, hasUpdate, releaseDate }: HeaderProps) { - return ( -
+ return (
- Twitter/X Media Batch Downloader window.location.reload()} - /> -

window.location.reload()} - > + Twitter/X Media Batch Downloader window.location.reload()}/> +

window.location.reload()}> Twitter/X Media Batch Downloader

- - {hasUpdate && releaseDate && ( - + {hasUpdate && releaseDate && (

{formatRelativeTime(releaseDate)}

-
- )} +
)}
- {hasUpdate && ( - + {hasUpdate && ( - - )} + )}
-
- ); +
); } diff --git a/frontend/src/components/MediaList.tsx b/frontend/src/components/MediaList.tsx index 6d7abe8..1668e9d 100644 --- a/frontend/src/components/MediaList.tsx +++ b/frontend/src/components/MediaList.tsx @@ -3,48 +3,9 @@ import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; import { Badge } from "@/components/ui/badge"; import { Progress } from "@/components/ui/progress"; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from "@/components/ui/select"; -import { - Tooltip, - TooltipContent, - TooltipTrigger, -} from "@/components/ui/tooltip"; - -import { - Image, - Video, - Film, - FileText, - ExternalLink, - Repeat2, - Download, - FolderOpen, - LayoutGrid, - Grid3X3, - List, - StopCircle, - Users, - UserPlus, - MessageSquare, - Calendar, - ArrowUp, - ChevronLeft, - ChevronRight, - Eye, - Heart, - Bookmark, - BadgeCheck, - Maximize2, - CheckCircle, - XCircle, - FileCheck, -} from "lucide-react"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue, } from "@/components/ui/select"; +import { Tooltip, TooltipContent, TooltipTrigger, } from "@/components/ui/tooltip"; +import { Image, Video, Film, FileText, ExternalLink, Repeat2, Download, FolderOpen, LayoutGrid, Grid3X3, List, StopCircle, Users, UserPlus, MessageSquare, Calendar, ArrowUp, ChevronLeft, ChevronRight, Eye, Heart, Bookmark, BadgeCheck, Maximize2, CheckCircle, XCircle, FileCheck, } from "lucide-react"; import { Spinner } from "@/components/ui/spinner"; import type { TimelineEntry, AccountInfo } from "@/types/api"; import { logger } from "@/lib/logger"; @@ -54,677 +15,587 @@ import { openExternal } from "@/lib/utils"; import { DownloadMediaWithMetadata, OpenFolder, IsFFmpegInstalled, ConvertGIFs, StopDownload, CheckFolderExists, CheckGifsFolderHasMP4 } from "../../wailsjs/go/main/App"; import { EventsOn, EventsOff } from "../../wailsjs/runtime/runtime"; import { main } from "../../wailsjs/go/models"; - interface DownloadProgress { - current: number; - total: number; - percent: number; + current: number; + total: number; + percent: number; } - interface DownloadItemStatus { - tweet_id: number; - index: number; - status: "success" | "failed" | "skipped"; + tweet_id: number; + index: number; + status: "success" | "failed" | "skipped"; } - interface MediaListProps { - accountInfo: AccountInfo; - timeline: TimelineEntry[]; - totalUrls: number; - fetchedMediaType?: string; - newMediaCount?: number | null; + accountInfo: AccountInfo; + timeline: TimelineEntry[]; + totalUrls: number; + fetchedMediaType?: string; + newMediaCount?: number | null; } - function getThumbnailUrl(url: string): string { - // For animated_gif: video.twimg.com/tweet_video/XXX.mp4 -> pbs.twimg.com/tweet_video_thumb/XXX?format=jpg&name=360x360 - if (url.includes("video.twimg.com/tweet_video/")) { - // Extract filename from URL (e.g., GzstJyDbIAAPuX9.mp4 -> GzstJyDbIAAPuX9) - const match = url.match(/tweet_video\/([^\/]+)\.mp4/); - if (match && match[1]) { - const filename = match[1]; - return `https://pbs.twimg.com/tweet_video_thumb/${filename}?format=jpg&name=360x360`; + if (url.includes("video.twimg.com/tweet_video/")) { + const match = url.match(/tweet_video\/([^\/]+)\.mp4/); + if (match && match[1]) { + const filename = match[1]; + return `https://pbs.twimg.com/tweet_video_thumb/${filename}?format=jpg&name=360x360`; + } } - } - - // For photos: pbs.twimg.com/media/XXX -> use 360x360 for consistency - if (url.includes("pbs.twimg.com/media/")) { - if (url.includes("?format=")) { - if (url.includes("&name=")) { - const parts = url.split("&name="); - return parts[0] + "&name=360x360"; - } - return url + "&name=360x360"; + if (url.includes("pbs.twimg.com/media/")) { + if (url.includes("?format=")) { + if (url.includes("&name=")) { + const parts = url.split("&name="); + return parts[0] + "&name=360x360"; + } + return url + "&name=360x360"; + } + if (url.includes("?")) { + return url + "&name=360x360"; + } + return url + "?format=jpg&name=360x360"; } - if (url.includes("?")) { - return url + "&name=360x360"; - } - return url + "?format=jpg&name=360x360"; - } - return url; + return url; } - function getPreviewUrl(url: string): string { - // For images, use large size for preview - if (url.includes("pbs.twimg.com/media/")) { - if (url.includes("?format=")) { - if (url.includes("&name=")) { - const parts = url.split("&name="); - return parts[0] + "&name=large"; - } - return url + "&name=large"; + if (url.includes("pbs.twimg.com/media/")) { + if (url.includes("?format=")) { + if (url.includes("&name=")) { + const parts = url.split("&name="); + return parts[0] + "&name=large"; + } + return url + "&name=large"; + } + if (url.includes("?")) { + return url + "&name=large"; + } + return url + "?format=jpg&name=large"; } - if (url.includes("?")) { - return url + "&name=large"; - } - return url + "?format=jpg&name=large"; - } - return url; + return url; } - function getMediaIcon(type: string) { - switch (type) { - case "photo": - return ; - case "video": - return