diff --git a/app.go b/app.go index 212223c..449b846 100644 --- a/app.go +++ b/app.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "path/filepath" + "sync" "twitterxmediabatchdownloader/backend" "github.com/wailsapp/wails/v2/pkg/runtime" @@ -13,8 +14,9 @@ import ( type App struct { ctx context.Context - downloadCtx context.Context + downloadMu sync.Mutex downloadCancel context.CancelFunc + downloadActive bool } func NewApp() *App { @@ -180,14 +182,22 @@ func (a *App) GetExtractorVersionStatus() backend.ExtractorVersionStatus { } func (a *App) Quit() { + if a.ctx != nil { + runtime.Quit(a.ctx) + return + } panic("quit") } type DownloadMediaRequest struct { - URLs []string `json:"urls"` - OutputDir string `json:"output_dir"` - Username string `json:"username"` - Proxy string `json:"proxy,omitempty"` + URLs []string `json:"urls"` + OutputDir string `json:"output_dir"` + Username string `json:"username"` + ConcurrentDownloads int `json:"concurrent_downloads,omitempty"` + SkipExisting bool `json:"skip_existing"` + DeleteIncompleteFiles bool `json:"delete_incomplete_files"` + RetryAttempts int `json:"retry_attempts,omitempty"` + Proxy string `json:"proxy,omitempty"` } type MediaItemRequest struct { @@ -201,10 +211,14 @@ type MediaItemRequest struct { } type DownloadMediaWithMetadataRequest struct { - Items []MediaItemRequest `json:"items"` - OutputDir string `json:"output_dir"` - Username string `json:"username"` - Proxy string `json:"proxy,omitempty"` + Items []MediaItemRequest `json:"items"` + OutputDir string `json:"output_dir"` + Username string `json:"username"` + ConcurrentDownloads int `json:"concurrent_downloads,omitempty"` + SkipExisting bool `json:"skip_existing"` + DeleteIncompleteFiles bool `json:"delete_incomplete_files"` + RetryAttempts int `json:"retry_attempts,omitempty"` + Proxy string `json:"proxy,omitempty"` } type DownloadMediaResponse struct { @@ -234,7 +248,14 @@ func (a *App) DownloadMedia(req DownloadMediaRequest) (DownloadMediaResponse, er outputDir = filepath.Join(outputDir, req.Username) } - downloaded, failed, err := backend.DownloadMediaFiles(req.URLs, outputDir, req.Proxy) + options := backend.DownloadOptions{ + ConcurrentDownloads: req.ConcurrentDownloads, + SkipExistingFiles: req.SkipExisting, + DeleteIncompleteFiles: req.DeleteIncompleteFiles, + RetryAttempts: req.RetryAttempts, + } + + downloaded, failed, err := backend.DownloadMediaFiles(req.URLs, outputDir, options, req.Proxy) if err != nil { return DownloadMediaResponse{ Success: false, @@ -307,11 +328,15 @@ func (a *App) DownloadMediaWithMetadata(req DownloadMediaWithMetadataRequest) (D } } - a.downloadCtx, a.downloadCancel = context.WithCancel(context.Background()) - defer func() { - a.downloadCtx = nil - a.downloadCancel = nil - }() + downloadCtx, err := a.startDownloadOperation() + if err != nil { + return DownloadMediaResponse{ + Success: false, + Cancelled: false, + Message: err.Error(), + }, err + } + defer a.finishDownloadOperation() progressCallback := func(current, total int) { percent := 0 @@ -333,7 +358,23 @@ func (a *App) DownloadMediaWithMetadata(req DownloadMediaWithMetadataRequest) (D }) } - downloaded, skipped, failed, err := backend.DownloadMediaWithMetadataProgressAndStatus(items, outputDir, req.Username, progressCallback, itemStatusCallback, a.downloadCtx, req.Proxy) + options := backend.DownloadOptions{ + ConcurrentDownloads: req.ConcurrentDownloads, + SkipExistingFiles: req.SkipExisting, + DeleteIncompleteFiles: req.DeleteIncompleteFiles, + RetryAttempts: req.RetryAttempts, + } + + downloaded, skipped, failed, err := backend.DownloadMediaWithMetadataProgressAndStatus( + items, + outputDir, + req.Username, + progressCallback, + itemStatusCallback, + downloadCtx, + options, + req.Proxy, + ) if err != nil { if errors.Is(err, context.Canceled) { return DownloadMediaResponse{ @@ -366,13 +407,39 @@ func (a *App) DownloadMediaWithMetadata(req DownloadMediaWithMetadataRequest) (D }, nil } -func (a *App) StopDownload() bool { - if a.downloadCancel != nil { - a.downloadCancel() - a.downloadCancel = nil - return true +func (a *App) startDownloadOperation() (context.Context, error) { + a.downloadMu.Lock() + defer a.downloadMu.Unlock() + + if a.downloadActive { + return nil, fmt.Errorf("another download is already in progress") } - return false + + ctx, cancel := context.WithCancel(context.Background()) + a.downloadCancel = cancel + a.downloadActive = true + + return ctx, nil +} + +func (a *App) finishDownloadOperation() { + a.downloadMu.Lock() + defer a.downloadMu.Unlock() + + a.downloadCancel = nil + a.downloadActive = false +} + +func (a *App) StopDownload() bool { + a.downloadMu.Lock() + defer a.downloadMu.Unlock() + + if !a.downloadActive || a.downloadCancel == nil { + return false + } + + a.downloadCancel() + return true } func (a *App) SaveAccountToDB(username, name, profileImage string, totalMedia int, responseJSON string, mediaType string) error { diff --git a/backend/cmd_unix.go b/backend/cmd_unix.go index 54359d9..9416a46 100644 --- a/backend/cmd_unix.go +++ b/backend/cmd_unix.go @@ -2,8 +2,26 @@ package backend -import "os/exec" +import ( + "errors" + "os/exec" + "syscall" +) func hideWindow(cmd *exec.Cmd) { - + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setpgid: true, + } +} + +func terminateCommandProcess(cmd *exec.Cmd) error { + if cmd == nil || cmd.Process == nil { + return nil + } + + if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil && !errors.Is(err, syscall.ESRCH) { + return err + } + + return nil } diff --git a/backend/cmd_windows.go b/backend/cmd_windows.go index 122d354..608badb 100644 --- a/backend/cmd_windows.go +++ b/backend/cmd_windows.go @@ -4,6 +4,7 @@ package backend import ( "os/exec" + "strconv" "syscall" ) @@ -13,3 +14,17 @@ func hideWindow(cmd *exec.Cmd) { CreationFlags: 0x08000000, } } + +func terminateCommandProcess(cmd *exec.Cmd) error { + if cmd == nil || cmd.Process == nil { + return nil + } + + taskkill := exec.Command("taskkill", "/T", "/F", "/PID", strconv.Itoa(cmd.Process.Pid)) + hideWindow(taskkill) + if err := taskkill.Run(); err == nil { + return nil + } + + return cmd.Process.Kill() +} diff --git a/backend/database.go b/backend/database.go index d747475..6392ce1 100644 --- a/backend/database.go +++ b/backend/database.go @@ -6,6 +6,8 @@ import ( "fmt" "os" "path/filepath" + "strings" + "sync" "time" _ "modernc.org/sqlite" @@ -41,6 +43,15 @@ type AccountListItem struct { } var db *sql.DB +var dbInitOnce sync.Once +var dbInitErr error + +func ensureDB() error { + dbInitOnce.Do(func() { + dbInitErr = initDBInternal() + }) + return dbInitErr +} type accountMetricsPayload struct { AccountInfo struct { @@ -60,6 +71,10 @@ func GetDBPath() string { } func InitDB() error { + return ensureDB() +} + +func initDBInternal() error { dbPath := GetDBPath() dir := filepath.Dir(dbPath) @@ -100,16 +115,12 @@ func InitDB() error { return err } - db.Exec("ALTER TABLE accounts ADD COLUMN group_name TEXT DEFAULT ''") - db.Exec("ALTER TABLE accounts ADD COLUMN group_color TEXT DEFAULT ''") - db.Exec("ALTER TABLE accounts ADD COLUMN media_type TEXT DEFAULT 'all'") - db.Exec("ALTER TABLE accounts ADD COLUMN cursor TEXT DEFAULT ''") - db.Exec("ALTER TABLE accounts ADD COLUMN completed INTEGER DEFAULT 1") - db.Exec("ALTER TABLE accounts ADD COLUMN followers_count INTEGER DEFAULT 0") - db.Exec("ALTER TABLE accounts ADD COLUMN statuses_count INTEGER DEFAULT 0") - db.Exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_username_media_type ON accounts(username, media_type)") + if err := runMigrations(); err != nil { + return err + } + if err := backfillAccountMetrics(); err != nil { return err } @@ -117,6 +128,37 @@ func InitDB() error { return nil } +func runMigrations() error { + var version int + if err := db.QueryRow("PRAGMA user_version").Scan(&version); err != nil { + return err + } + + migrations := []string{ + "ALTER TABLE accounts ADD COLUMN group_name TEXT DEFAULT ''", + "ALTER TABLE accounts ADD COLUMN group_color TEXT DEFAULT ''", + "ALTER TABLE accounts ADD COLUMN media_type TEXT DEFAULT 'all'", + "ALTER TABLE accounts ADD COLUMN cursor TEXT DEFAULT ''", + "ALTER TABLE accounts ADD COLUMN completed INTEGER DEFAULT 1", + "ALTER TABLE accounts ADD COLUMN followers_count INTEGER DEFAULT 0", + "ALTER TABLE accounts ADD COLUMN statuses_count INTEGER DEFAULT 0", + } + + for i := version; i < len(migrations); i++ { + if _, err := db.Exec(migrations[i]); err != nil { + if !strings.Contains(err.Error(), "duplicate column name") { + return fmt.Errorf("migration %d failed: %w", i, err) + } + } + } + + if _, err := db.Exec(fmt.Sprintf("PRAGMA user_version = %d", len(migrations))); err != nil { + return err + } + + return nil +} + func CloseDB() { if db != nil { db.Close() @@ -128,10 +170,8 @@ func SaveAccount(username, name, profileImage string, totalMedia int, responseJS } func SaveAccountWithStatus(username, name, profileImage string, totalMedia int, responseJSON string, mediaType string, cursor string, completed bool) error { - if db == nil { - if err := InitDB(); err != nil { - return err - } + if err := ensureDB(); err != nil { + return err } if mediaType == "" { @@ -164,10 +204,8 @@ func SaveAccountWithStatus(username, name, profileImage string, totalMedia int, } func GetAllAccounts() ([]AccountListItem, error) { - if db == nil { - if err := InitDB(); err != nil { - return nil, err - } + if err := ensureDB(); err != nil { + return nil, err } rows, err := db.Query(` @@ -264,10 +302,8 @@ func backfillAccountMetrics() error { } func UpdateAccountGroup(id int64, groupName, groupColor string) error { - if db == nil { - if err := InitDB(); err != nil { - return err - } + if err := ensureDB(); err != nil { + return err } _, err := db.Exec("UPDATE accounts SET group_name = ?, group_color = ? WHERE id = ?", groupName, groupColor, id) @@ -275,10 +311,8 @@ func UpdateAccountGroup(id int64, groupName, groupColor string) error { } func GetAllGroups() ([]map[string]string, error) { - if db == nil { - if err := InitDB(); err != nil { - return nil, err - } + if err := ensureDB(); err != nil { + return nil, err } rows, err := db.Query(` @@ -305,10 +339,8 @@ func GetAllGroups() ([]map[string]string, error) { } func ClearAllAccounts() error { - if db == nil { - if err := InitDB(); err != nil { - return err - } + if err := ensureDB(); err != nil { + return err } _, err := db.Exec("DELETE FROM accounts") @@ -316,10 +348,8 @@ func ClearAllAccounts() error { } func GetAccountByID(id int64) (*AccountDB, error) { - if db == nil { - if err := InitDB(); err != nil { - return nil, err - } + if err := ensureDB(); err != nil { + return nil, err } var acc AccountDB @@ -345,10 +375,8 @@ func GetAccountByID(id int64) (*AccountDB, error) { } func GetAccountByUsernameAndMediaType(username, mediaType string) (*AccountDB, error) { - if db == nil { - if err := InitDB(); err != nil { - return nil, err - } + if err := ensureDB(); err != nil { + return nil, err } if mediaType == "" { @@ -380,10 +408,8 @@ func GetAccountByUsernameAndMediaType(username, mediaType string) (*AccountDB, e } func DeleteAccount(id int64) error { - if db == nil { - if err := InitDB(); err != nil { - return err - } + if err := ensureDB(); err != nil { + return err } _, err := db.Exec("DELETE FROM accounts WHERE id = ?", id) @@ -523,13 +549,7 @@ func ExportAccountsToTXT(ids []int64, outputDir string) (string, error) { return "", fmt.Errorf("no valid usernames found") } - txtContent := "" - for i, username := range usernames { - if i > 0 { - txtContent += "\n" - } - txtContent += username - } + txtContent := strings.Join(usernames, "\n") filePath := filepath.Join(exportDir, "twitterxmediabatchdownloader_multiple.txt") @@ -541,10 +561,8 @@ func ExportAccountsToTXT(ids []int64, outputDir string) (string, error) { } func ImportAccountFromFile(filePath string) (string, error) { - if db == nil { - if err := InitDB(); err != nil { - return "", err - } + if err := ensureDB(); err != nil { + return "", err } data, err := os.ReadFile(filePath) diff --git a/backend/download.go b/backend/download.go index 6d60ef8..ac76bc7 100644 --- a/backend/download.go +++ b/backend/download.go @@ -2,6 +2,7 @@ package backend import ( "context" + "errors" "fmt" "io" "net/http" @@ -15,9 +16,45 @@ import ( ) const ( - MaxConcurrentDownloads = 10 + DefaultConcurrentDownloads = 10 + MaxConcurrentDownloads = 50 + DefaultRetryAttempts = 2 + MaxRetryAttempts = 5 ) +func NormalizeConcurrentDownloads(requested int) int { + if requested <= 0 { + return DefaultConcurrentDownloads + } + if requested > MaxConcurrentDownloads { + return MaxConcurrentDownloads + } + return requested +} + +func NormalizeRetryAttempts(requested int) int { + if requested < 0 { + return 0 + } + if requested > MaxRetryAttempts { + return MaxRetryAttempts + } + return requested +} + +type DownloadOptions struct { + ConcurrentDownloads int + SkipExistingFiles bool + DeleteIncompleteFiles bool + RetryAttempts int +} + +func NormalizeDownloadOptions(options DownloadOptions) DownloadOptions { + options.ConcurrentDownloads = NormalizeConcurrentDownloads(options.ConcurrentDownloads) + options.RetryAttempts = NormalizeRetryAttempts(options.RetryAttempts) + return options +} + type MediaItem struct { URL string `json:"url"` Date string `json:"date"` @@ -28,7 +65,8 @@ type MediaItem struct { OriginalFilename string `json:"original_filename,omitempty"` } -func DownloadMediaFiles(urls []string, outputDir string, customProxy string) (downloaded int, failed int, err error) { +func DownloadMediaFiles(urls []string, outputDir string, options DownloadOptions, customProxy string) (downloaded int, failed int, err error) { + options = NormalizeDownloadOptions(options) if err := os.MkdirAll(outputDir, 0755); err != nil { return 0, len(urls), fmt.Errorf("failed to create output directory: %v", err) @@ -46,12 +84,14 @@ func DownloadMediaFiles(urls []string, outputDir string, customProxy string) (do filename := extractFilename(mediaURL) outputPath := filepath.Join(outputDir, filename) - if _, err := os.Stat(outputPath); err == nil { - downloaded++ - continue + if options.SkipExistingFiles { + if _, err := os.Stat(outputPath); err == nil { + downloaded++ + continue + } } - if err := downloadFile(client, mediaURL, outputPath); err != nil { + if err := downloadFileWithRetry(context.Background(), client, mediaURL, outputPath, options); err != nil { failed++ continue } @@ -61,6 +101,140 @@ func DownloadMediaFiles(urls []string, outputDir string, customProxy string) (do return downloaded, failed, nil } +func fileExists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + return false, err +} + +func cleanupIncompleteDownload(outputPath string) { + _ = os.Remove(outputPath) +} + +func replaceFile(tempPath, outputPath string, allowOverwrite bool) error { + if !allowOverwrite { + return os.Rename(tempPath, outputPath) + } + + exists, err := fileExists(outputPath) + if err != nil { + return err + } + if !exists { + return os.Rename(tempPath, outputPath) + } + + backupPath := fmt.Sprintf("%s.bak.%d", outputPath, time.Now().UnixNano()) + if err := os.Rename(outputPath, backupPath); err != nil { + return err + } + + if err := os.Rename(tempPath, outputPath); err != nil { + restoreErr := os.Rename(backupPath, outputPath) + if restoreErr != nil { + return fmt.Errorf("failed to replace existing file: %v (restore failed: %v)", err, restoreErr) + } + return err + } + + _ = os.Remove(backupPath) + return nil +} + +func writeFileAtomically(outputPath string, data []byte, allowOverwrite bool, keepPartialOnFailure bool) error { + out, err := os.CreateTemp(filepath.Dir(outputPath), filepath.Base(outputPath)+".*.part") + if err != nil { + return err + } + tempPath := out.Name() + cleanupTemp := !keepPartialOnFailure + defer func() { + if out != nil { + _ = out.Close() + } + if cleanupTemp { + _ = os.Remove(tempPath) + } + }() + + if _, err := out.Write(data); err != nil { + return err + } + + if err := out.Close(); err != nil { + return err + } + out = nil + + if err := replaceFile(tempPath, outputPath, allowOverwrite); err != nil { + return err + } + + cleanupTemp = false + return nil +} + +func waitForRetry(ctx context.Context, attempt int) error { + timer := time.NewTimer(time.Duration(attempt) * 500 * time.Millisecond) + defer timer.Stop() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func downloadFileWithRetry(ctx context.Context, client *http.Client, mediaURL, outputPath string, options DownloadOptions) error { + options = NormalizeDownloadOptions(options) + + var lastErr error + totalAttempts := options.RetryAttempts + 1 + for attempt := 1; attempt <= totalAttempts; attempt++ { + keepPartialOnFailure := !options.DeleteIncompleteFiles && attempt == totalAttempts + lastErr = downloadFileWithContext(ctx, client, mediaURL, outputPath, !options.SkipExistingFiles, keepPartialOnFailure) + if lastErr == nil { + return nil + } + if ctx.Err() != nil || errors.Is(lastErr, context.Canceled) { + if options.DeleteIncompleteFiles { + cleanupIncompleteDownload(outputPath) + } + return lastErr + } + if options.DeleteIncompleteFiles { + cleanupIncompleteDownload(outputPath) + } + if attempt < totalAttempts { + if err := waitForRetry(ctx, attempt); err != nil { + if options.DeleteIncompleteFiles { + cleanupIncompleteDownload(outputPath) + } + return err + } + } + } + + return lastErr +} + +func writeTextFileWithOptions(outputPath string, content string, options DownloadOptions) error { + keepPartialOnFailure := !options.DeleteIncompleteFiles + if err := writeFileAtomically(outputPath, []byte(content), !options.SkipExistingFiles, keepPartialOnFailure); err != nil { + if options.DeleteIncompleteFiles { + cleanupIncompleteDownload(outputPath) + } + return err + } + return nil +} + type ProgressCallback func(current, total int) type ItemStatusCallback func(tweetID int64, index int, status string) // status: "success", "failed", "skipped" @@ -71,10 +245,11 @@ type downloadTask struct { index int } -func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir string, username string, progress ProgressCallback, itemStatus ItemStatusCallback, ctx context.Context, customProxy string) (downloaded int, skipped int, failed int, err error) { +func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir string, username string, progress ProgressCallback, itemStatus ItemStatusCallback, ctx context.Context, options DownloadOptions, customProxy string) (downloaded int, skipped int, failed int, err error) { if ctx == nil { ctx = context.Background() } + options = NormalizeDownloadOptions(options) total := len(items) if total == 0 { @@ -144,7 +319,7 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str taskChan := make(chan downloadTask, len(tasks)) var wg sync.WaitGroup - numWorkers := MaxConcurrentDownloads + numWorkers := options.ConcurrentDownloads if numWorkers > len(tasks) { numWorkers = len(tasks) } @@ -167,14 +342,6 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str client := sharedClient for task := range taskChan { - - select { - case <-ctx.Done(): - return - default: - } - - var status string markCompleted := func() { completed := atomic.AddInt64(&completedCount, 1) if progress != nil { @@ -182,25 +349,54 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str } } - if _, err := os.Stat(task.outputPath); err == nil { - status = "skipped" - + if ctx.Err() != nil { if itemStatus != nil { - itemStatus(task.item.TweetID, task.index, status) + itemStatus(task.item.TweetID, task.index, "cancelled") } - atomic.AddInt64(&skippedCount, 1) markCompleted() continue - } else if task.item.Type == "text" { + } - if err := os.WriteFile(task.outputPath, []byte(task.item.Content), 0644); err != nil { + var status string + + if options.SkipExistingFiles { + exists, err := fileExists(task.outputPath) + if err == nil && exists { + status = "skipped" + + if itemStatus != nil { + itemStatus(task.item.TweetID, task.index, status) + } + atomic.AddInt64(&skippedCount, 1) + markCompleted() + continue + } + } + + if task.item.Type == "text" { + if ctx.Err() != nil { + if itemStatus != nil { + itemStatus(task.item.TweetID, task.index, "cancelled") + } + markCompleted() + continue + } + + if err := writeTextFileWithOptions(task.outputPath, task.item.Content, options); err != nil { atomic.AddInt64(&failedCount, 1) status = "failed" } else { atomic.AddInt64(&downloadedCount, 1) status = "success" } - } else if err := downloadFileWithContext(ctx, client, task.item.URL, task.outputPath); err != nil { + } else if err := downloadFileWithRetry(ctx, client, task.item.URL, task.outputPath, options); err != nil { + if ctx.Err() != nil || errors.Is(err, context.Canceled) { + if itemStatus != nil { + itemStatus(task.item.TweetID, task.index, "cancelled") + } + markCompleted() + continue + } atomic.AddInt64(&failedCount, 1) status = "failed" } else { @@ -209,8 +405,10 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str originalFilename := ExtractOriginalFilename(task.item.URL) - if err := EmbedMetadata(task.outputPath, task.item.Content, tweetURL, originalFilename); err != nil { + if ctx.Err() == nil { + if err := EmbedMetadata(ctx, task.outputPath, task.item.Content, tweetURL, originalFilename); err != nil && !errors.Is(err, context.Canceled) { + } } atomic.AddInt64(&downloadedCount, 1) @@ -231,7 +429,7 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str case <-ctx.Done(): close(taskChan) wg.Wait() - return int(downloadedCount), int(skippedCount), int(failedCount) + (total - int(completedCount)), ctx.Err() + return int(downloadedCount), int(skippedCount), int(failedCount), ctx.Err() case taskChan <- task: } } @@ -239,10 +437,14 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str wg.Wait() + if err := ctx.Err(); err != nil { + return int(downloadedCount), int(skippedCount), int(failedCount), err + } + return int(downloadedCount), int(skippedCount), int(failedCount), nil } -func downloadFileWithContext(ctx context.Context, client *http.Client, url, outputPath string) error { +func downloadFileWithContext(ctx context.Context, client *http.Client, url, outputPath string, allowOverwrite bool, keepPartialOnFailure bool) error { req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { return err @@ -258,14 +460,41 @@ func downloadFileWithContext(ctx context.Context, client *http.Client, url, outp return fmt.Errorf("bad status: %s", resp.Status) } - out, err := os.Create(outputPath) + out, err := os.CreateTemp(filepath.Dir(outputPath), filepath.Base(outputPath)+".*.part") if err != nil { return err } - defer out.Close() + tempPath := out.Name() + cleanupTemp := !keepPartialOnFailure + defer func() { + if out != nil { + _ = out.Close() + } + if cleanupTemp { + _ = os.Remove(tempPath) + } + }() _, err = io.Copy(out, resp.Body) - return err + if err != nil { + return err + } + + if err := out.Close(); err != nil { + return err + } + out = nil + + if ctx.Err() != nil { + return ctx.Err() + } + + if err := replaceFile(tempPath, outputPath, allowOverwrite); err != nil { + return err + } + + cleanupTemp = false + return nil } func formatTimestamp(dateStr string) string { @@ -354,22 +583,9 @@ func extractFilename(mediaURL string) string { } func downloadFile(client *http.Client, url, outputPath string) error { - resp, err := client.Get(url) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("bad status: %s", resp.Status) - } - - out, err := os.Create(outputPath) - if err != nil { - return err - } - defer out.Close() - - _, err = io.Copy(out, resp.Body) - return err + return downloadFileWithRetry(context.Background(), client, url, outputPath, DownloadOptions{ + SkipExistingFiles: false, + DeleteIncompleteFiles: true, + RetryAttempts: 0, + }) } diff --git a/backend/exiftool.go b/backend/exiftool.go index 5a4b6e8..ba8ec7b 100644 --- a/backend/exiftool.go +++ b/backend/exiftool.go @@ -469,6 +469,10 @@ func extractExifToolFromZip(zipPath, destPath string) error { } libDir := filepath.Join(filepath.Dir(destPath), "exiftool_files") + libDirAbs, err := filepath.Abs(libDir) + if err != nil { + return fmt.Errorf("failed to resolve libDir: %v", err) + } for _, f := range r.File { if strings.Contains(f.Name, "exiftool_files") && !strings.HasSuffix(f.Name, "/") { parts := strings.Split(f.Name, "exiftool_files/") @@ -482,6 +486,14 @@ func extractExifToolFromZip(zipPath, destPath string) error { targetPath := filepath.Join(libDir, relPath) + targetAbs, err := filepath.Abs(targetPath) + if err != nil { + continue + } + if !strings.HasPrefix(targetAbs+string(os.PathSeparator), libDirAbs+string(os.PathSeparator)) && targetAbs != libDirAbs { + continue + } + if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil { continue } diff --git a/backend/folder.go b/backend/folder.go index 2bf4a01..2a4b731 100644 --- a/backend/folder.go +++ b/backend/folder.go @@ -11,12 +11,19 @@ import ( ) func OpenFolderInExplorer(path string) error { + info, err := os.Stat(path) + if err != nil { + return err + } + if !info.IsDir() { + return os.ErrInvalid + } + var cmd *exec.Cmd switch runtime.GOOS { case "windows": - - cmd = exec.Command("cmd", "/c", "start", "", path) + cmd = exec.Command("explorer", path) case "darwin": cmd = exec.Command("open", path) case "linux": diff --git a/backend/metadata.go b/backend/metadata.go index bf323bd..29d0fed 100644 --- a/backend/metadata.go +++ b/backend/metadata.go @@ -1,6 +1,7 @@ package backend import ( + "context" "fmt" "net/url" "os" @@ -11,6 +12,11 @@ import ( "strings" ) +var ( + filenameSafeRegex = regexp.MustCompile(`^[A-Za-z0-9_]+$`) + filenameLetterRegex = regexp.MustCompile(`[A-Za-z]`) +) + func ExtractOriginalFilename(mediaURL string) string { parsedURL, err := url.Parse(mediaURL) if err != nil { @@ -29,7 +35,7 @@ func ExtractOriginalFilename(mediaURL string) string { filename = filename[:idx] } - if matched, _ := regexp.MatchString(`^[A-Za-z0-9_]+$`, filename); matched && len(filename) > 0 { + if filenameSafeRegex.MatchString(filename) && len(filename) > 0 { return filename } } @@ -44,7 +50,7 @@ func ExtractOriginalFilename(mediaURL string) string { filename = filename[:idx] } - if matched, _ := regexp.MatchString(`^[A-Za-z0-9_]+$`, filename); matched && len(filename) > 0 { + if filenameSafeRegex.MatchString(filename) && len(filename) > 0 { return filename } } @@ -58,9 +64,8 @@ func ExtractOriginalFilename(mediaURL string) string { filename = filename[:idx] } - if matched, _ := regexp.MatchString(`^[A-Za-z0-9_]+$`, filename); matched { - hasLetter, _ := regexp.MatchString(`[A-Za-z]`, filename) - if hasLetter && len(filename) >= 8 { + if filenameSafeRegex.MatchString(filename) { + if filenameLetterRegex.MatchString(filename) && len(filename) >= 8 { return filename } } @@ -70,21 +75,24 @@ func ExtractOriginalFilename(mediaURL string) string { return "" } -func EmbedMetadata(filePath string, tweetContent string, tweetURL string, originalFilename string) error { +func EmbedMetadata(ctx context.Context, filePath string, tweetContent string, tweetURL string, originalFilename string) error { ext := strings.ToLower(filepath.Ext(filePath)) switch ext { case ".jpg", ".jpeg": - return embedImageMetadata(filePath, tweetContent, tweetURL, originalFilename) + return embedImageMetadata(ctx, filePath, tweetContent, tweetURL, originalFilename) case ".mp4": - return embedVideoMetadata(filePath, tweetContent, tweetURL, originalFilename) + return embedVideoMetadata(ctx, filePath, tweetContent, tweetURL, originalFilename) default: return nil } } -func embedImageMetadata(filePath string, tweetContent string, tweetURL string, originalFilename string) error { +func embedImageMetadata(ctx context.Context, filePath string, tweetContent string, tweetURL string, originalFilename string) error { + if ctx == nil { + ctx = context.Background() + } exiftoolPath := findExifTool() if exiftoolPath == "" { @@ -100,10 +108,13 @@ func embedImageMetadata(filePath string, tweetContent string, tweetURL string, o filePath, } - cmd := exec.Command(exiftoolPath, args...) + cmd := exec.CommandContext(ctx, exiftoolPath, args...) hideWindow(cmd) output, err := cmd.CombinedOutput() if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } return fmt.Errorf("exiftool error (non-fatal): %v, output: %s", err, string(output)) } @@ -112,7 +123,10 @@ func embedImageMetadata(filePath string, tweetContent string, tweetURL string, o return nil } -func embedVideoMetadata(filePath string, tweetContent string, tweetURL string, originalFilename string) error { +func embedVideoMetadata(ctx context.Context, filePath string, tweetContent string, tweetURL string, originalFilename string) error { + if ctx == nil { + ctx = context.Background() + } exiftoolPath := findExifTool() if exiftoolPath == "" { @@ -120,10 +134,13 @@ func embedVideoMetadata(filePath string, tweetContent string, tweetURL string, o return nil } - return embedVideoMetadataWithExifTool(exiftoolPath, filePath, tweetContent, tweetURL, originalFilename) + return embedVideoMetadataWithExifTool(ctx, exiftoolPath, filePath, tweetContent, tweetURL, originalFilename) } -func embedVideoMetadataWithExifTool(exiftoolPath string, filePath string, tweetContent string, tweetURL string, originalFilename string) error { +func embedVideoMetadataWithExifTool(ctx context.Context, exiftoolPath string, filePath string, tweetContent string, tweetURL string, originalFilename string) error { + if ctx == nil { + ctx = context.Background() + } metadataComment := buildMetadataComment(tweetContent, tweetURL, originalFilename) @@ -133,10 +150,13 @@ func embedVideoMetadataWithExifTool(exiftoolPath string, filePath string, tweetC filePath, } - cmd := exec.Command(exiftoolPath, args...) + cmd := exec.CommandContext(ctx, exiftoolPath, args...) hideWindow(cmd) output, err := cmd.CombinedOutput() if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } return fmt.Errorf("exiftool error (non-fatal): %v, output: %s", err, string(output)) } diff --git a/backend/twitter.go b/backend/twitter.go index 64f53b3..fe2585d 100644 --- a/backend/twitter.go +++ b/backend/twitter.go @@ -82,7 +82,7 @@ func KillAllExtractorProcesses() { for _, cmd := range commands { if cmd != nil && cmd.Process != nil { - _ = cmd.Process.Kill() + _ = terminateCommandProcess(cmd) } } } @@ -548,6 +548,9 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) { if err != nil { outputStr := string(output) + if strings.TrimSpace(outputStr) == "" { + return nil, fmt.Errorf("xtractor process terminated before returning data") + } errorMsg := parseExtractorError(outputStr, req.Username) return nil, fmt.Errorf("%s", errorMsg) } @@ -731,6 +734,9 @@ func ExtractDateRange(req DateRangeRequest) (*TwitterResponse, error) { if err != nil { outputStr := string(output) + if strings.TrimSpace(outputStr) == "" { + return nil, fmt.Errorf("xtractor process terminated before returning data") + } errorMsg := parseExtractorError(outputStr, req.Username) return nil, fmt.Errorf("%s", errorMsg) } diff --git a/frontend/pnpm-workspace.yaml b/frontend/pnpm-workspace.yaml new file mode 100644 index 0000000..93d4aa8 --- /dev/null +++ b/frontend/pnpm-workspace.yaml @@ -0,0 +1,3 @@ +allowBuilds: + esbuild: true + sharp: true \ No newline at end of file diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 7edf16c..976a9ae 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -2,8 +2,10 @@ import { useState, useEffect, useRef, useCallback } from "react"; import { TooltipProvider } from "@/components/ui/tooltip"; import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; import { Button } from "@/components/ui/button"; +import { ArrowLeft } from "lucide-react"; import { getSettings, applyThemeMode, applyFont } from "@/lib/settings"; import { applyTheme } from "@/lib/themes"; +import { openExternal } from "@/lib/utils"; import { compareVersionNumbers } from "@/lib/version"; import { toastWithSound as toast } from "@/lib/toast-with-sound"; import { logger } from "@/lib/logger"; @@ -92,18 +94,54 @@ function App() { toast.warning("Fetch completed, but saving to Saved Accounts failed"); } }; + const restorePartialFetchState = async (cleanUsername: string, mediaType: string, showStoppedToast: boolean) => { + const savedState = getFetchState(cleanUsername); + if (!savedState || savedState.totalFetched <= 0) { + if (showStoppedToast) { + toast.info("Fetch stopped"); + } + return; + } + const partialResponse = await loadSavedAccountResponse(cleanUsername, savedState.mediaType || mediaType); + setResumeInfo({ canResume: true, mediaCount: savedState.totalFetched }); + if (partialResponse) { + setResult(partialResponse); + try { + await SaveAccountToDBWithStatus(partialResponse.account_info.name, partialResponse.account_info.nick, partialResponse.account_info.profile_image, partialResponse.total_urls, JSON.stringify(partialResponse), savedState.mediaType || mediaType, savedState.cursor || "", false); + } + catch (dbErr) { + reportDatabaseSaveError("partial fetch data", dbErr); + } + } + if (showStoppedToast) { + toast.info(`Stopped at ${formatNumberWithComma(savedState.totalFetched)} items`); + } + }; + const stopMultipleFetches = (showToast: boolean) => { + stopAllRef.current = true; + setIsFetchingAll(false); + accountTimersRef.current.forEach((interval) => clearInterval(interval)); + accountTimersRef.current.clear(); + accountStartTimesRef.current.clear(); + accountTimeoutSecondsRef.current.clear(); + setMultipleAccounts((prev) => prev.map((acc) => { + if (acc.status === "fetching") { + const mediaCount = accountMediaCountRef.current.get(acc.id) || acc.mediaCount || 0; + return { ...acc, status: "incomplete" as const, mediaCount, remainingTime: 0 }; + } + return acc; + })); + CleanupExtractorProcesses().catch(() => { }); + if (showToast) { + toast.info("Stopped all fetches"); + } + }; const handleFetchTypeChange = (type: FetchType) => { setFetchType(type); if (type === "single") { - setMultipleAccounts([]); - setIsFetchingAll(false); - stopAllRef.current = true; - accountTimersRef.current.forEach((interval) => clearInterval(interval)); - accountTimersRef.current.clear(); - accountStartTimesRef.current.clear(); - accountTimeoutSecondsRef.current.clear(); - accountStopFlagsRef.current.clear(); - accountMediaCountRef.current.clear(); + if (isFetchingAll || multipleAccounts.some((acc) => acc.status === "fetching")) { + stopMultipleFetches(false); + } } else { setResult(null); @@ -129,6 +167,40 @@ function App() { mediaQuery.removeEventListener("change", handleChange); }; }, []); + useEffect(() => { + const handleDocumentClick = (event: MouseEvent) => { + if (event.defaultPrevented || event.button !== 0 || event.metaKey || event.ctrlKey || event.shiftKey || event.altKey) { + return; + } + const target = event.target; + if (!(target instanceof Element)) { + return; + } + const anchor = target.closest("a[href]"); + if (!(anchor instanceof HTMLAnchorElement)) { + return; + } + const href = anchor.getAttribute("href"); + if (!href || href.startsWith("#") || href.startsWith("javascript:")) { + return; + } + try { + const url = new URL(href, window.location.href); + if (url.protocol !== "http:" && url.protocol !== "https:") { + return; + } + event.preventDefault(); + openExternal(url.toString()); + } + catch { + // Ignore invalid URLs and allow the browser/webview to handle them. + } + }; + document.addEventListener("click", handleDocumentClick); + return () => { + document.removeEventListener("click", handleDocumentClick); + }; + }, []); const checkForUpdates = async () => { try { const response = await fetch("https://api.github.com/repos/afkarxyz/Twitter-X-Media-Batch-Downloader/releases/latest"); @@ -559,6 +631,11 @@ function App() { catch (error) { const errorMsg = error instanceof Error ? error.message : String(error); const elapsedSecs = fetchStartTimeRef.current ? Math.floor((Date.now() - fetchStartTimeRef.current) / 1000) : 0; + if (stopFetchRef.current) { + logger.info(`Stopped fetch after ${elapsedSecs}s`); + await restorePartialFetchState(cleanUsername, requestedMediaType, true); + return; + } logger.error(`Failed to fetch: ${errorMsg} (${elapsedSecs}s)`); if (errorMsg.toLowerCase().includes("xtractor dependency") || errorMsg.toLowerCase().includes("extractor dependency")) { toast.error(errorMsg); @@ -568,18 +645,8 @@ function App() { } const savedState = getFetchState(cleanUsername); if (savedState && savedState.totalFetched > 0) { - const partialResponse = await loadSavedAccountResponse(cleanUsername, savedState.mediaType || requestedMediaType); - if (partialResponse) { - setResult(partialResponse); - setResumeInfo({ canResume: true, mediaCount: savedState.totalFetched }); - toast.info(`Saved ${formatNumberWithComma(savedState.totalFetched)} items - can resume`); - try { - await SaveAccountToDBWithStatus(partialResponse.account_info.name, partialResponse.account_info.nick, partialResponse.account_info.profile_image, partialResponse.total_urls, JSON.stringify(partialResponse), savedState.mediaType || requestedMediaType, savedState.cursor || "", false); - } - catch (dbErr) { - reportDatabaseSaveError("partial fetch data", dbErr); - } - } + await restorePartialFetchState(cleanUsername, requestedMediaType, false); + toast.info(`Saved ${formatNumberWithComma(savedState.totalFetched)} items - can resume`); } } finally { @@ -628,6 +695,13 @@ function App() { toast.error("Failed to parse saved data"); } }; + const handleBackToHomeResult = () => { + setCurrentPage("main"); + setResult(null); + setNewMediaCount(null); + const scrollElement = document.getElementById("app-content-scroll"); + scrollElement?.scrollTo({ top: 0, behavior: "smooth" }); + }; const parseUsername = (input: string): string => { let clean = input.trim(); if (clean.startsWith("@")) { @@ -667,43 +741,59 @@ function App() { setCurrentPage("main"); toast.success(`Added ${formatNumberWithComma(accounts.length)} account(s) to multiple fetch`); }; - const handleImportFile = () => { - const input = document.createElement("input"); - input.type = "file"; - input.accept = ".txt"; - input.onchange = async (e) => { - const file = (e.target as HTMLInputElement).files?.[0]; - if (!file) - return; - try { - const text = await file.text(); - const lines = text.split("\n").map((line) => line.trim()).filter((line) => line.length > 0); - if (lines.length === 0) { - toast.error("File is empty"); - return; - } - const accounts: MultipleAccount[] = lines.map((line) => { - const username = parseUsername(line); - return { - id: crypto.randomUUID(), - username, - status: "pending", - mediaCount: 0, - previousMediaCount: 0, - elapsedTime: 0, - remainingTime: null, - showDiff: false, - }; - }); - setMultipleAccounts(accounts); - toast.success(`Imported ${formatNumberWithComma(accounts.length)} account(s)`); + const handleImportAccounts = (entries: string[]) => { + const seenUsernames = new Set(); + const uniqueUsernames = entries + .map((entry) => parseUsername(entry)) + .filter((entry) => entry.length > 0) + .filter((entry) => { + const normalized = entry.toLowerCase(); + if (seenUsernames.has(normalized)) { + return false; } - catch (error) { - console.error("Failed to read imported account list:", error); - toast.error("Failed to read file"); - } - }; - input.click(); + seenUsernames.add(normalized); + return true; + }); + if (uniqueUsernames.length === 0) { + toast.error("No valid accounts found"); + return; + } + const accounts: MultipleAccount[] = uniqueUsernames.map((entry) => ({ + id: crypto.randomUUID(), + username: entry, + status: "pending", + mediaCount: 0, + previousMediaCount: 0, + elapsedTime: 0, + remainingTime: null, + showDiff: false, + })); + setMultipleAccounts(accounts); + toast.success(`Imported ${formatNumberWithComma(accounts.length)} account(s)`); + }; + const handleClearMultipleAccounts = () => { + accountTimersRef.current.forEach((interval) => clearInterval(interval)); + accountTimersRef.current.clear(); + accountStartTimesRef.current.clear(); + accountTimeoutSecondsRef.current.clear(); + accountStopFlagsRef.current.clear(); + accountMediaCountRef.current.clear(); + stopAllRef.current = false; + setIsFetchingAll(false); + setMultipleAccounts([]); + toast.info("Multiple fetch list cleared"); + }; + const handleRemoveMultipleAccount = (accountId: string) => { + const account = multipleAccounts.find((entry) => entry.id === accountId); + accountTimersRef.current.delete(accountId); + accountStartTimesRef.current.delete(accountId); + accountTimeoutSecondsRef.current.delete(accountId); + accountStopFlagsRef.current.delete(accountId); + accountMediaCountRef.current.delete(accountId); + setMultipleAccounts((prev) => prev.filter((entry) => entry.id !== accountId)); + if (account) { + toast.info(`Removed @${account.username} from list`); + } }; const handleFetchAll = async (authToken: string) => { if (multipleAccounts.length === 0) { @@ -960,6 +1050,19 @@ function App() { accountTimeoutSecondsRef.current.delete(accountId); const errorMsg = error instanceof Error ? error.message : String(error); const mediaCount = accountMediaCountRef.current.get(accountId) || 0; + if (stopAllRef.current || accountStopFlagsRef.current.get(accountId)) { + accountMediaCountRef.current.delete(accountId); + setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId + ? { + ...acc, + status: "incomplete" as const, + error: undefined, + mediaCount, + } + : acc)); + logger.info(`@${account.username}: stopped - ${mediaCount} items (${elapsedSecs}s)`); + continue; + } const isAuthError = errorMsg.toLowerCase().includes("401") || errorMsg.toLowerCase().includes("unauthorized") || errorMsg.toLowerCase().includes("auth token may be invalid") || @@ -984,30 +1087,14 @@ function App() { } }; const handleStopAll = () => { - stopAllRef.current = true; - setIsFetchingAll(false); - accountTimersRef.current.forEach((interval) => clearInterval(interval)); - accountTimersRef.current.clear(); - accountStartTimesRef.current.clear(); - accountTimeoutSecondsRef.current.clear(); - setMultipleAccounts((prev) => prev.map((acc) => { - if (acc.status === "fetching") { - const mediaCount = accountMediaCountRef.current.get(acc.id) || acc.mediaCount || 0; - const status = mediaCount === 0 ? ("failed" as const) : ("incomplete" as const); - return { ...acc, status, mediaCount }; - } - return acc; - })); - CleanupExtractorProcesses().catch(() => { }); - toast.info("Stopped all fetches"); + stopMultipleFetches(true); }; const handleStopAccount = (accountId: string) => { accountStopFlagsRef.current.set(accountId, true); setMultipleAccounts((prev) => prev.map((acc) => { if (acc.id === accountId && acc.status === "fetching") { const mediaCount = accountMediaCountRef.current.get(accountId) || acc.mediaCount || 0; - const status = mediaCount === 0 ? ("failed" as const) : ("incomplete" as const); - return { ...acc, status, mediaCount }; + return { ...acc, status: "incomplete" as const, mediaCount }; } return acc; })); @@ -1301,6 +1388,19 @@ function App() { accountTimeoutSecondsRef.current.delete(accountId); const errorMsg = error instanceof Error ? error.message : String(error); const mediaCount = accountMediaCountRef.current.get(accountId) || 0; + if (stopAllRef.current || accountStopFlagsRef.current.get(accountId)) { + accountMediaCountRef.current.delete(accountId); + setMultipleAccounts((prev) => prev.map((acc) => acc.id === accountId + ? { + ...acc, + status: "incomplete" as const, + error: undefined, + mediaCount, + } + : acc)); + logger.info(`@${account.username}: stopped - ${mediaCount} items (${elapsedSecs}s)`); + return; + } const isAuthError = errorMsg.toLowerCase().includes("401") || errorMsg.toLowerCase().includes("unauthorized") || errorMsg.toLowerCase().includes("auth token may be invalid") || @@ -1333,14 +1433,20 @@ function App() { return (<>
- { + { setSearchMode(mode); if (privateType) { setSearchPrivateType(privateType); } }}/> - {result && fetchType === "single" && (
+ {result && fetchType === "single" && (
+
+ +
)} ); @@ -1367,7 +1473,7 @@ function App() { } handleCancelNavigation(); }}> - + Unsaved Changes diff --git a/frontend/src/components/DatabaseView.tsx b/frontend/src/components/DatabaseView.tsx index 979c03f..f7e3c8d 100644 --- a/frontend/src/components/DatabaseView.tsx +++ b/frontend/src/components/DatabaseView.tsx @@ -1,4 +1,4 @@ -import { useState, useEffect, useRef, useCallback, useMemo } from "react"; +import { useState, useEffect, useRef, useCallback, useMemo, useDeferredValue } from "react"; import { Button } from "@/components/ui/button"; import { Badge } from "@/components/ui/badge"; import { Input } from "@/components/ui/input"; @@ -16,7 +16,7 @@ import { toastWithSound as toast } from "@/lib/toast-with-sound"; import { getSettings } from "@/lib/settings"; import { openExternal } from "@/lib/utils"; import { GetAllAccountsFromDB, GetAccountFromDB, DeleteAccountFromDB, SaveAccountToDB, ExportAccountJSON, ExportAccountsTXT, UpdateAccountGroup, GetAllGroups, DownloadMediaWithMetadata, StopDownload, CheckFoldersExist, OpenFolder, GetFolderPath, } from "../../wailsjs/go/main/App"; -import { EventsOn, EventsOff } from "../../wailsjs/runtime/runtime"; +import { EventsOn } from "../../wailsjs/runtime/runtime"; import { main } from "../../wailsjs/go/models"; interface DownloadProgress { current: number; @@ -82,6 +82,7 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr const [filterMediaType, setFilterMediaType] = useState("all"); const [accountViewMode, setAccountViewMode] = useState<"public" | "private">("public"); const [searchQuery, setSearchQuery] = useState(""); + const deferredSearchQuery = useDeferredValue(searchQuery); const [sortOrder, setSortOrder] = useState<"newest" | "oldest" | "username-asc" | "username-desc" | "followers-high" | "followers-low" | "posts-high" | "posts-low" | "media-high" | "media-low">("newest"); const [gridView, setGridView] = useState<"large" | "small" | "list">("list"); const [editingAccount, setEditingAccount] = useState(null); @@ -147,7 +148,6 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr setDownloadProgress(progress); }); return () => { - EventsOff("download-progress"); unsubscribe(); }; }, []); @@ -162,8 +162,8 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr const baseAccounts = accountViewMode === "public" ? publicAccounts : privateAccounts; return baseAccounts .filter((acc) => { - if (searchQuery) { - const query = searchQuery.toLowerCase(); + if (deferredSearchQuery) { + const query = deferredSearchQuery.toLowerCase(); const matchesUsername = acc.username.toLowerCase().includes(query); const matchesName = acc.name.toLowerCase().includes(query); if (!matchesUsername && !matchesName) @@ -219,7 +219,7 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr return 0; } }); - }, [accountViewMode, filterGroup, filterMediaType, privateAccounts, publicAccounts, searchQuery, sortOrder]); + }, [accountViewMode, filterGroup, filterMediaType, privateAccounts, publicAccounts, deferredSearchQuery, sortOrder]); const visibleAccounts = useMemo(() => filteredAccounts.slice(0, visibleCount), [filteredAccounts, visibleCount]); const visibleAccountsKey = visibleAccounts .map((account) => `${account.id}:${account.username}`) @@ -377,6 +377,10 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr })), output_dir: outputDir, username: actualUsername, + concurrent_downloads: settings.concurrentDownloads || 10, + skip_existing: settings.skipExistingFiles, + delete_incomplete_files: settings.deleteIncompleteFiles, + retry_attempts: settings.retryAttempts, proxy: settings.proxy || "", }); const response = await DownloadMediaWithMetadata(request); @@ -496,6 +500,10 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr })), output_dir: outputDir, username: actualUsername, + concurrent_downloads: settings.concurrentDownloads || 10, + skip_existing: settings.skipExistingFiles, + delete_incomplete_files: settings.deleteIncompleteFiles, + retry_attempts: settings.retryAttempts, proxy: settings.proxy || "", }); const response = await DownloadMediaWithMetadata(request); @@ -611,7 +619,15 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr input.type = "file"; input.accept = ".json"; input.multiple = true; + input.style.display = "none"; + document.body.appendChild(input); + const cleanup = () => { + if (input.parentNode) { + input.parentNode.removeChild(input); + } + }; input.onchange = async (e) => { + try { const files = (e.target as HTMLInputElement).files; if (!files || files.length === 0) return; @@ -710,6 +726,9 @@ export function DatabaseView({ onLoadAccount, onUpdateSelected }: DatabaseViewPr else { toast.error("No valid files imported"); } + } finally { + cleanup(); + } }; input.click(); }; diff --git a/frontend/src/components/Header.tsx b/frontend/src/components/Header.tsx index 3a928e7..4e5f487 100644 --- a/frontend/src/components/Header.tsx +++ b/frontend/src/components/Header.tsx @@ -11,8 +11,8 @@ export function Header({ version, hasUpdate, releaseDate }: HeaderProps) { return (
- Twitter/X Media Batch Downloader window.location.reload()}/> -

window.location.reload()}> + Twitter/X Media Batch Downloader +

Twitter/X Media Batch Downloader

diff --git a/frontend/src/components/MediaList.tsx b/frontend/src/components/MediaList.tsx index 59f50db..5ff664b 100644 --- a/frontend/src/components/MediaList.tsx +++ b/frontend/src/components/MediaList.tsx @@ -14,7 +14,7 @@ import { getSettings } from "@/lib/settings"; import { getCachedDependencyStatus, getCachedMediaFolderStatus, setCachedDependencyStatus, setCachedMediaFolderStatus } from "@/lib/runtime-cache"; import { openExternal } from "@/lib/utils"; import { DownloadMediaWithMetadata, OpenFolder, IsFFmpegInstalled, ConvertGIFs, StopDownload, CheckFolderExists, CheckGifsFolderHasMP4 } from "../../wailsjs/go/main/App"; -import { EventsOn, EventsOff } from "../../wailsjs/runtime/runtime"; +import { EventsOn } from "../../wailsjs/runtime/runtime"; import { main } from "../../wailsjs/go/models"; interface DownloadProgress { current: number; @@ -185,7 +185,7 @@ export function MediaList({ accountInfo, timeline, totalUrls, fetchedMediaType = const cachedMediaFolderStatus = settings.downloadPath ? getCachedMediaFolderStatus(settings.downloadPath, folderName) : null; - const [selectedItems, setSelectedItems] = useState>(new Set()); + const [selectedItems, setSelectedItems] = useState>(new Set()); const [sortBy, setSortBy] = useState("date-desc"); const [filterType, setFilterType] = useState("all"); const [viewMode, setViewMode] = useState<"large" | "small" | "list">("list"); @@ -301,14 +301,13 @@ export function MediaList({ accountInfo, timeline, totalUrls, fetchedMediaType = setPreviewIndex(null); }; useEffect(() => { - if (previewIndex !== null) { - document.body.style.overflow = "hidden"; - } - else { - document.body.style.overflow = ""; + if (previewIndex === null) { + return; } + const previousOverflow = document.body.style.overflow; + document.body.style.overflow = "hidden"; return () => { - document.body.style.overflow = ""; + document.body.style.overflow = previousOverflow; }; }, [previewIndex]); useEffect(() => { @@ -316,7 +315,6 @@ export function MediaList({ accountInfo, timeline, totalUrls, fetchedMediaType = setDownloadProgress(progress); }); return () => { - EventsOff("download-progress"); unsubscribe(); }; }, []); @@ -361,7 +359,6 @@ export function MediaList({ accountInfo, timeline, totalUrls, fetchedMediaType = } }); return () => { - EventsOff("download-item-status"); unsubscribe(); }; }, []); @@ -408,16 +405,16 @@ export function MediaList({ accountInfo, timeline, totalUrls, fetchedMediaType = setSelectedItems(new Set()); } else { - setSelectedItems(new Set(filteredTimeline.map((_, i) => i))); + setSelectedItems(new Set(filteredTimeline.map((item) => getItemKey(item)))); } }; - const toggleItem = (index: number) => { + const toggleItem = (key: string) => { const newSelected = new Set(selectedItems); - if (newSelected.has(index)) { - newSelected.delete(index); + if (newSelected.has(key)) { + newSelected.delete(key); } else { - newSelected.add(index); + newSelected.add(key); } setSelectedItems(newSelected); }; @@ -434,7 +431,9 @@ export function MediaList({ accountInfo, timeline, totalUrls, fetchedMediaType = }; const handleDownload = async () => { const itemsWithIndices = selectedItems.size > 0 - ? Array.from(selectedItems).map((i) => ({ item: filteredTimeline[i], originalIndex: i })) + ? filteredTimeline + .map((item, i) => ({ item, originalIndex: i })) + .filter((entry) => selectedItems.has(getItemKey(entry.item))) : filteredTimeline.map((item, i) => ({ item, originalIndex: i })); if (itemsWithIndices.length === 0) { toast.error("No media to download"); @@ -462,6 +461,10 @@ export function MediaList({ accountInfo, timeline, totalUrls, fetchedMediaType = })), output_dir: getOutputDir(), username: accountInfo.name, + concurrent_downloads: settings.concurrentDownloads || 10, + skip_existing: settings.skipExistingFiles, + delete_incomplete_files: settings.deleteIncompleteFiles, + retry_attempts: settings.retryAttempts, proxy: settings.proxy || "", }); const response = await DownloadMediaWithMetadata(request); @@ -774,14 +777,14 @@ export function MediaList({ accountInfo, timeline, totalUrls, fetchedMediaType = {viewMode === "list" ? (
{filteredTimeline.slice(0, visibleCount).map((item, index) => { - const isSelected = selectedItems.has(index); const itemKey = getItemKey(item); + const isSelected = selectedItems.has(itemKey); const isItemDownloaded = downloadedItems.has(itemKey); const isItemFailed = failedItems.has(itemKey); const isItemSkipped = skippedItems.has(itemKey); const isItemDownloading = downloadingItem === itemKey; return (
- toggleItem(index)}/> + toggleItem(itemKey)}/> {index + 1} @@ -833,6 +836,10 @@ export function MediaList({ accountInfo, timeline, totalUrls, fetchedMediaType = })], output_dir: getOutputDir(), username: accountInfo.name, + concurrent_downloads: settings.concurrentDownloads || 10, + skip_existing: settings.skipExistingFiles, + delete_incomplete_files: settings.deleteIncompleteFiles, + retry_attempts: settings.retryAttempts, proxy: settings.proxy || "", }); const response = await DownloadMediaWithMetadata(request); @@ -872,8 +879,8 @@ export function MediaList({ accountInfo, timeline, totalUrls, fetchedMediaType = })}
) : (
{filteredTimeline.slice(0, visibleCount).map((item, index) => { - const isSelected = selectedItems.has(index); const itemKey = getItemKey(item); + const isSelected = selectedItems.has(itemKey); const isItemDownloaded = downloadedItems.has(itemKey); const isItemFailed = failedItems.has(itemKey); const isItemSkipped = skippedItems.has(itemKey); @@ -906,6 +913,10 @@ export function MediaList({ accountInfo, timeline, totalUrls, fetchedMediaType = })], output_dir: getOutputDir(), username: accountInfo.name, + concurrent_downloads: settings.concurrentDownloads || 10, + skip_existing: settings.skipExistingFiles, + delete_incomplete_files: settings.deleteIncompleteFiles, + retry_attempts: settings.retryAttempts, proxy: settings.proxy || "", }); const response = await DownloadMediaWithMetadata(request); @@ -947,7 +958,7 @@ export function MediaList({ accountInfo, timeline, totalUrls, fetchedMediaType =
e.stopPropagation()}> - toggleItem(index)} className="bg-background/80"/> + toggleItem(itemKey)} className="bg-background/80"/>
@@ -1079,6 +1090,10 @@ export function MediaList({ accountInfo, timeline, totalUrls, fetchedMediaType = })], output_dir: getOutputDir(), username: accountInfo.name, + concurrent_downloads: settings.concurrentDownloads || 10, + skip_existing: settings.skipExistingFiles, + delete_incomplete_files: settings.deleteIncompleteFiles, + retry_attempts: settings.retryAttempts, proxy: settings.proxy || "", }); const response = await DownloadMediaWithMetadata(request); diff --git a/frontend/src/components/SearchBar.tsx b/frontend/src/components/SearchBar.tsx index 4a040cf..ab7ade9 100644 --- a/frontend/src/components/SearchBar.tsx +++ b/frontend/src/components/SearchBar.tsx @@ -1,27 +1,33 @@ -import { useState, useEffect } from "react"; +import { useEffect, useRef, useState } from "react"; import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; +import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; import { InputWithContext } from "@/components/ui/input-with-context"; import { Label } from "@/components/ui/label"; -import { XCircle, Calendar, StopCircle, Globe, Lock, Bookmark, Heart, Cookie, Eye, EyeOff, ChevronDown, ChevronUp, Info, RotateCcw, Trash2, Clock, CloudDownload, User, Users, FileText, Hourglass, CheckCircle, AlertCircle, SlidersHorizontal, } from "lucide-react"; -import { Spinner } from "@/components/ui/spinner"; -function formatNumberWithComma(num: number): string { - return num.toLocaleString(); -} -import { Checkbox } from "@/components/ui/checkbox"; -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue, } from "@/components/ui/select"; -import { Tooltip, TooltipContent, TooltipTrigger, } from "@/components/ui/tooltip"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { FetchHistory } from "@/components/FetchHistory"; import type { HistoryItem } from "@/components/FetchHistory"; +import { toastWithSound as toast } from "@/lib/toast-with-sound"; import { cn } from "@/lib/utils"; import { getSettings, updateSettings, type FetchMode as SettingsFetchMode, type MediaType as SettingsMediaType } from "@/lib/settings"; import { GetStoredAuthToken, SetStoredAuthToken } from "../../wailsjs/go/main/App"; +import { AlertCircle, Bookmark, Calendar, CheckCircle, CloudDownload, Clock, Eye, EyeOff, Globe, Heart, Hourglass, Info, Lock, RotateCcw, SlidersHorizontal, StopCircle, Trash2, User, Users, XCircle } from "lucide-react"; +import { Spinner } from "@/components/ui/spinner"; + +function formatNumberWithComma(num: number): string { + return num.toLocaleString(); +} + export type FetchMode = "public" | "private"; export type PrivateType = "bookmarks" | "likes"; export type FetchType = "single" | "multiple"; + const LEGACY_PUBLIC_AUTH_TOKEN_KEY = "twitter_public_auth_token"; const LEGACY_PRIVATE_AUTH_TOKEN_KEY = "twitter_private_auth_token"; -let sessionPublicAuthToken = ""; -let sessionPrivateAuthToken = ""; + +let sessionAuthToken = ""; + export interface MultipleAccount { id: string; username: string; @@ -39,6 +45,7 @@ export interface MultipleAccount { showDiff?: boolean; cursor?: string; } + interface SearchBarProps { username: string; loading: boolean; @@ -60,42 +67,62 @@ interface SearchBarProps { fetchType?: FetchType; onFetchTypeChange?: (type: FetchType) => void; multipleAccounts?: MultipleAccount[]; - onImportFile?: () => void; + onImportAccounts?: (accounts: string[]) => void; onFetchAll?: (authToken: string) => void; onStopAll?: () => void; onStopAccount?: (accountId: string) => void; onRetryAccount?: (accountId: string, authToken: string) => void; + onClearMultipleAccounts?: () => void; + onRemoveMultipleAccount?: (accountId: string) => void; isFetchingAll?: boolean; mode?: FetchMode; privateType?: PrivateType; onModeChange?: (mode: FetchMode, privateType?: PrivateType) => void; } -export function SearchBar({ username, loading, onUsernameChange, onFetch, onStopFetch, onResume, onClearResume, resumeInfo, history, onHistorySelect, onHistoryRemove, hasResult, elapsedTime = 0, remainingTime = null, fetchType = "single", onFetchTypeChange, multipleAccounts = [], onImportFile, onFetchAll, onStopAll, onStopAccount, onRetryAccount, isFetchingAll = false, mode: externalMode, privateType: externalPrivateType, onModeChange, }: SearchBarProps) { + +export function SearchBar({ username, loading, onUsernameChange, onFetch, onStopFetch, onResume, onClearResume, resumeInfo, history, onHistorySelect, onHistoryRemove, hasResult, elapsedTime = 0, remainingTime = null, fetchType = "single", onFetchTypeChange, multipleAccounts = [], onImportAccounts, onFetchAll, onStopAll, onStopAccount, onRetryAccount, onClearMultipleAccounts, onRemoveMultipleAccount, isFetchingAll = false, mode: externalMode, privateType: externalPrivateType, onModeChange, }: SearchBarProps) { const [useDateRange, setUseDateRange] = useState(false); const [startDate, setStartDate] = useState(""); const [endDate, setEndDate] = useState(""); const [mediaType, setMediaType] = useState(getSettings().mediaType); const [retweets, setRetweets] = useState(getSettings().includeRetweets); + const [currentFetchMode, setCurrentFetchMode] = useState(getSettings().fetchMode); + const [sharedAuthToken, setSharedAuthToken] = useState(() => sessionAuthToken || localStorage.getItem(LEGACY_PUBLIC_AUTH_TOKEN_KEY) || localStorage.getItem(LEGACY_PRIVATE_AUTH_TOKEN_KEY) || ""); + const [showAuthToken, setShowAuthToken] = useState(false); + const [showAuthRequiredDialog, setShowAuthRequiredDialog] = useState(false); + const [showSettingsDialog, setShowSettingsDialog] = useState(false); + useEffect(() => { + if (showSettingsDialog) { + const fresh = getSettings(); + setMediaType(fresh.mediaType); + setRetweets(fresh.includeRetweets); + setCurrentFetchMode(fresh.fetchMode); + } + }, [showSettingsDialog]); + const [showImportDialog, setShowImportDialog] = useState(false); + const [importAccountsText, setImportAccountsText] = useState(""); + const [authPromptToken, setAuthPromptToken] = useState(""); + const mode = externalMode || "public"; const privateType = externalPrivateType || "bookmarks"; - const [showAuthInput, setShowAuthInput] = useState(false); - const [showAdvanced, setShowAdvanced] = useState(false); - const [currentFetchMode, setCurrentFetchMode] = useState(getSettings().fetchMode); - const [publicAuthToken, setPublicAuthToken] = useState(() => sessionPublicAuthToken || localStorage.getItem(LEGACY_PUBLIC_AUTH_TOKEN_KEY) || ""); - const [privateAuthToken, setPrivateAuthToken] = useState(() => sessionPrivateAuthToken || localStorage.getItem(LEGACY_PRIVATE_AUTH_TOKEN_KEY) || ""); - const [showPublicToken, setShowPublicToken] = useState(false); - const [showPrivateToken, setShowPrivateToken] = useState(false); + const isLikesMode = mode === "private" && privateType === "likes"; + const isBookmarksMode = mode === "private" && privateType === "bookmarks"; + const isAnyMultipleAccountFetching = multipleAccounts.some((account) => account.status === "fetching"); + useEffect(() => { let active = true; + const syncStoredTokens = async () => { const legacyPublicToken = localStorage.getItem(LEGACY_PUBLIC_AUTH_TOKEN_KEY) || ""; const legacyPrivateToken = localStorage.getItem(LEGACY_PRIVATE_AUTH_TOKEN_KEY) || ""; + const legacyUnifiedToken = legacyPublicToken || legacyPrivateToken; + try { - if (legacyPublicToken) { - await SetStoredAuthToken("public", legacyPublicToken); - } - if (legacyPrivateToken) { - await SetStoredAuthToken("private", legacyPrivateToken); + if (legacyUnifiedToken) { + await Promise.all([ + SetStoredAuthToken("public", legacyUnifiedToken), + SetStoredAuthToken("private", legacyUnifiedToken), + ]); } } catch (error) { @@ -114,80 +141,177 @@ export function SearchBar({ username, loading, onUsernameChange, onFetch, onStop if (!active) { return; } - sessionPublicAuthToken = storedPublicToken || ""; - setPublicAuthToken(storedPublicToken || ""); - sessionPrivateAuthToken = storedPrivateToken || ""; - setPrivateAuthToken(storedPrivateToken || ""); + + const unifiedToken = storedPublicToken || storedPrivateToken || ""; + sessionAuthToken = unifiedToken; + setSharedAuthToken(unifiedToken); + + if (unifiedToken && (storedPublicToken !== unifiedToken || storedPrivateToken !== unifiedToken)) { + await Promise.all([ + SetStoredAuthToken("public", unifiedToken), + SetStoredAuthToken("private", unifiedToken), + ]); + } } catch (error) { console.error("Failed to load auth token from secure storage:", error); } }; + void syncStoredTokens(); + return () => { active = false; }; }, []); - const handlePublicTokenChange = async (value: string) => { - sessionPublicAuthToken = value; - setPublicAuthToken(value); - try { - await SetStoredAuthToken("public", value); + + const tokenWriteTimerRef = useRef | null>(null); + const pendingAuthActionRef = useRef<((authToken: string) => void) | null>(null); + useEffect(() => { + return () => { + if (tokenWriteTimerRef.current) { + clearTimeout(tokenWriteTimerRef.current); + } + }; + }, []); + + const handleSharedTokenChange = (value: string) => { + sessionAuthToken = value; + setSharedAuthToken(value); + if (tokenWriteTimerRef.current) { + clearTimeout(tokenWriteTimerRef.current); } - catch (error) { - console.error("Failed to store public auth token securely:", error); + tokenWriteTimerRef.current = setTimeout(() => { + void Promise.all([ + SetStoredAuthToken("public", value), + SetStoredAuthToken("private", value), + ]).catch((error) => { + console.error("Failed to store auth token securely:", error); + }); + }, 400); + }; + + const closeAuthRequiredDialog = (open: boolean) => { + setShowAuthRequiredDialog(open); + if (!open) { + pendingAuthActionRef.current = null; } }; - const handlePrivateTokenChange = async (value: string) => { - sessionPrivateAuthToken = value; - setPrivateAuthToken(value); - try { - await SetStoredAuthToken("private", value); - } - catch (error) { - console.error("Failed to store private auth token securely:", error); + + const requireAuthToken = (onAuthorized: (authToken: string) => void) => { + const token = sharedAuthToken.trim(); + if (token) { + onAuthorized(token); + return; } + pendingAuthActionRef.current = onAuthorized; + setAuthPromptToken(sharedAuthToken); + setShowAuthRequiredDialog(true); }; + const handleFetch = () => { - const authToken = mode === "public" ? publicAuthToken : privateAuthToken; - onFetch(useDateRange, startDate, endDate, mediaType, retweets, mode, privateType, authToken, false); + requireAuthToken((authToken) => { + onFetch(useDateRange, startDate, endDate, mediaType, retweets, mode, privateType, authToken, false); + }); }; + const handleResume = () => { - const authToken = mode === "public" ? publicAuthToken : privateAuthToken; if (onResume) { - onResume(authToken, mediaType, retweets); + requireAuthToken((authToken) => { + onResume(authToken, mediaType, retweets); + }); } }; - const currentAuthToken = mode === "public" ? publicAuthToken : privateAuthToken; - const hasAuthToken = currentAuthToken.trim().length > 0; - const isLikesMode = mode === "private" && privateType === "likes"; - const isBookmarksMode = mode === "private" && privateType === "bookmarks"; + + const handleFetchAll = () => { + requireAuthToken((authToken) => { + onFetchAll?.(authToken); + }); + }; + + const handleRetry = (accountId: string) => { + requireAuthToken((authToken) => { + onRetryAccount?.(accountId, authToken); + }); + }; + + const handleAuthPromptSubmit = () => { + const token = authPromptToken.trim(); + if (!token) { + toast.error("Please enter your auth token"); + return; + } + handleSharedTokenChange(token); + const pendingAction = pendingAuthActionRef.current; + pendingAuthActionRef.current = null; + setShowAuthRequiredDialog(false); + pendingAction?.(token); + }; + + const handleOpenFetchSettings = () => { + pendingAuthActionRef.current = null; + setShowAuthRequiredDialog(false); + setShowSettingsDialog(true); + }; + + const openImportDialog = () => { + setImportAccountsText(multipleAccounts.map((account) => account.username).join("\n")); + setShowImportDialog(true); + }; + + const handleImportSubmit = () => { + const accounts = importAccountsText + .split(/[\n,;]+/) + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0); + + if (accounts.length === 0) { + toast.error("Please enter at least one account"); + return; + } + + onImportAccounts?.(accounts); + setShowImportDialog(false); + }; + const formatTime = (seconds: number): string => { const mins = Math.floor(seconds / 60); const secs = seconds % 60; return `${mins}:${String(secs).padStart(2, "0")}`; }; + + const activeCountdown = remainingTime !== null && remainingTime >= 0 + ? formatTime(remainingTime) + : elapsedTime > 0 + ? formatTime(elapsedTime) + : null; + const fetchingLabel = activeCountdown ? `Fetching... (${activeCountdown})` : "Fetching..."; + const authTokenHelpText = mode === "private" + ? "Use auth token from the account whose bookmarks or likes you want to fetch." + : "Recommended to use a dummy account, not your main account. Excessive usage may cause suspension."; + return (
- +
-
- -
- {fetchType === "single" && (
+ + {fetchType === "single" && (
)}
- {fetchType === "multiple" && (
- + - +

- One username per line, example: -
- masteraoko -
- xbatchdemo -
- takomayuyi + One username or URL per line. Commas and semicolons are supported too.

-
- - - {isFetchingAll && ()} + +
+ {isFetchingAll && ()} + + + + + + Fetch Settings + + + +
)} - {fetchType === "single" && (mode === "public" || isLikesMode) && (
- +
onUsernameChange(e.target.value)} onKeyDown={(e) => e.key === "Enter" && handleFetch()} className="pr-8"/> - {username && ()}
+
- {loading && (<> - - {(remainingTime !== null || elapsedTime > 0) && (
- - - {remainingTime !== null && remainingTime >= 0 - ? `${Math.floor(remainingTime / 60)}:${String(remainingTime % 60).padStart(2, "0")}` - : `${Math.floor(elapsedTime / 60)}:${String(elapsedTime % 60).padStart(2, "0")}`} - -
)} - - )} - + {loading && ()} + {!loading && resumeInfo?.canResume && mode === "public" && (<> @@ -289,15 +405,26 @@ export function SearchBar({ username, loading, onUsernameChange, onFetch, onStop Clear Resume - )} - + + Fetch Settings + + +
)} -
- - - - - - - {mode === "private" && fetchType === "single" && (<> + )} - - {isBookmarksMode && (
- {loading && (<> - - {(remainingTime !== null || elapsedTime > 0) && (
- - - {remainingTime !== null && remainingTime >= 0 - ? `${Math.floor(remainingTime / 60)}:${String(remainingTime % 60).padStart(2, "0")}` - : `${Math.floor(elapsedTime / 60)}:${String(elapsedTime % 60).padStart(2, "0")}`} - -
)} - )} + + + + - )} -
)}
- - {showAuthInput && (
-
-
- - - - - - - {mode === "private" ? (

Use auth token from the account whose bookmarks/likes you want to fetch

) : (

Recommended to use a dummy account, not your main account.
Excessive usage may cause suspension

)} -
-
-
-
- mode === "public" - ? handlePublicTokenChange(e.target.value) - : handlePrivateTokenChange(e.target.value)} className="pr-10 bg-background"/> - -
-
-
)} - - - {showAdvanced && (
- -
- -
- - -
- - -
- - -
- - -
- { - const value = checked as boolean; - updateSettings({ includeRetweets: value }); - setRetweets(value); - }} className="bg-background"/> - -
- - - {mode === "public" && (<> -
- setUseDateRange(checked as boolean)} className="bg-background"/> - -
- - - {useDateRange && (<> - setStartDate(e.target.value)} className="w-35 h-8 bg-background"/> - - - setEndDate(e.target.value)} className="w-35 h-8 bg-background"/> - )} - )} -
-
)} - - - {fetchType === "multiple" && multipleAccounts.length > 0 && (
+ {fetchType === "multiple" && multipleAccounts.length > 0 && (
@@ -494,6 +491,7 @@ export function SearchBar({ username, loading, onUsernameChange, onFetch, onStop {formatNumberWithComma(multipleAccounts.filter((acc) => acc.status === "completed").length)}
+
Incomplete: @@ -501,6 +499,7 @@ export function SearchBar({ username, loading, onUsernameChange, onFetch, onStop {formatNumberWithComma(multipleAccounts.filter((acc) => acc.status === "incomplete").length)}
+
Failed: @@ -508,7 +507,8 @@ export function SearchBar({ username, loading, onUsernameChange, onFetch, onStop {formatNumberWithComma(multipleAccounts.filter((acc) => acc.status === "failed").length)}
-
+ +
Total: {formatNumberWithComma(multipleAccounts.filter((acc) => acc.status === "completed" || acc.status === "incomplete" || acc.status === "failed").length)}/{formatNumberWithComma(multipleAccounts.length)} @@ -517,26 +517,32 @@ export function SearchBar({ username, loading, onUsernameChange, onFetch, onStop
)} - {fetchType === "multiple" && multipleAccounts.length > 0 && (
-

Accounts to Fetch ({formatNumberWithComma(multipleAccounts.length)})

-
- {multipleAccounts.map((account) => (
+
+

Accounts to Fetch ({formatNumberWithComma(multipleAccounts.length)})

+ +
+
+ {multipleAccounts.map((account) => (
- {account.accountInfo ? ({account.accountInfo.nick}) : (
+ {account.accountInfo ? ({account.accountInfo.nick}) : (
)} -
+ +
{account.accountInfo ? (<>
{account.accountInfo.nick} @{account.accountInfo.name}
-
+
{account.mediaCount.toLocaleString()} items found - {account.showDiff && account.previousMediaCount > 0 && account.mediaCount > account.previousMediaCount && ( + {account.showDiff && account.previousMediaCount > 0 && account.mediaCount > account.previousMediaCount && ( +{account.mediaCount - account.previousMediaCount} )}
@@ -544,14 +550,15 @@ export function SearchBar({ username, loading, onUsernameChange, onFetch, onStop @{account.username}
)}
+
{account.status === "fetching" && (<> -
- +
+ {account.remainingTime !== null && account.remainingTime >= 0 - ? formatTime(account.remainingTime) - : formatTime(account.elapsedTime)} + ? formatTime(account.remainingTime) + : formatTime(account.elapsedTime)}
)} - {account.status === "pending" && ( + + {account.status === "pending" && ( Pending )} - {account.status === "completed" && ( + + {account.status === "completed" && ( Completed )} + {account.status === "incomplete" && (<> - + Incomplete - )} + {account.status === "failed" && (<> - + Failed - )} + +
))} @@ -594,5 +610,180 @@ export function SearchBar({ username, loading, onUsernameChange, onFetch, onStop
)} {!hasResult && mode === "public" && fetchType === "single" && ()} + + + + + Input Account List + + Enter one username or URL per line. Commas and semicolons also work. + + + +