diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml
new file mode 100644
index 0000000..75452fa
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.yml
@@ -0,0 +1,68 @@
+name: Bug Report
+description: Report a bug in Twitter/X Media Batch Downloader
+title: "[Bug Report] "
+labels: ["bug"]
+
+body:
+ - type: markdown
+ attributes:
+ value: |
+ > **WARNING: Issues that do not follow this template will be deleted without review.**
+ >
+ > **Please keep `[Bug Report]` in the issue title and only continue after it.**
+
+ - type: textarea
+ id: problem
+ attributes:
+ label: Problem
+ placeholder: e.g. Fetching @username/media stops after a few items, or GIF conversion fails with no clear error.
+ validations:
+ required: true
+
+ - type: input
+ id: target
+ attributes:
+ label: X/Twitter URL or Username
+ placeholder: e.g. https://x.com/username/media or @username
+ validations:
+ required: true
+
+ - type: textarea
+ id: steps
+ attributes:
+ label: Steps to Reproduce
+ placeholder: |
+ 1. Open ...
+ 2. Enter ...
+ 3. Click ...
+ 4. See error ...
+ validations:
+ required: true
+
+ - type: textarea
+ id: logs
+ attributes:
+ label: Error Message / Logs / Screenshot
+ placeholder: e.g. failed to extract timeline: parse_error ...
+ validations:
+ required: true
+
+ - type: markdown
+ attributes:
+ value: "### Environment"
+
+ - type: input
+ id: version
+ attributes:
+ label: Twitter/X Media Batch Downloader Version
+ placeholder: e.g. v4.4
+ validations:
+ required: true
+
+ - type: input
+ id: os
+ attributes:
+ label: OS
+ placeholder: e.g. Windows 11 23H2
+ validations:
+ required: true
diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
new file mode 100644
index 0000000..ec4bb38
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/config.yml
@@ -0,0 +1 @@
+blank_issues_enabled: false
\ No newline at end of file
diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml
new file mode 100644
index 0000000..f48e1a5
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.yml
@@ -0,0 +1,34 @@
+name: Feature Request
+description: Request a feature for Twitter/X Media Batch Downloader
+title: "[Feature Request] "
+labels: ["enhancement"]
+
+body:
+ - type: markdown
+ attributes:
+ value: |
+ > **WARNING: Issues that do not follow this template will be deleted without review.**
+ >
+ > **Please keep `[Feature Request]` in the issue title and only continue after it.**
+
+ - type: textarea
+ id: description
+ attributes:
+ label: Description
+ placeholder: e.g. Add an option to rotate between multiple auth tokens or export filtered results directly from Saved Accounts.
+ validations:
+ required: true
+
+ - type: textarea
+ id: use-case
+ attributes:
+ label: Why is this needed?
+ placeholder: e.g. I manage multiple accounts and want a faster way to retry only failed fetches without re-running the whole batch.
+ validations:
+ required: true
+
+ - type: textarea
+ id: additional-context
+ attributes:
+ label: Additional Context
+ placeholder: e.g. Mockup, similar app behavior, workflow notes, or screenshots if needed.
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..4664703
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,59 @@
+# Wails Build
+build/
+*.exe
+*.dll
+*.dylib
+*.so
+
+# Wails Generated Files
+frontend/wailsjs/
+
+# Go
+*.test
+*.out
+go.work
+go.work.sum
+
+# Node / Frontend
+node_modules/
+frontend/node_modules/
+frontend/dist/
+frontend/.vite/
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+.pnpm-store/
+.npm
+.yarn
+*.tsbuildinfo
+
+# IDE / Editors
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+.DS_Store
+
+# OS
+Thumbs.db
+desktop.ini
+
+# Environment
+.env
+.env.local
+.env.*.local
+
+# Logs
+*.log
+logs/
+
+# Temporary files
+tmp/
+temp/
+*.tmp
+*.bak
+*.old
+
+*.bat
diff --git a/app.go b/app.go
index d4afc7a..212223c 100644
--- a/app.go
+++ b/app.go
@@ -3,6 +3,7 @@ package main
import (
"context"
"encoding/json"
+ "errors"
"fmt"
"path/filepath"
"twitterxmediabatchdownloader/backend"
@@ -154,6 +155,18 @@ func (a *App) GetDefaults() map[string]string {
}
}
+func (a *App) GetStoredAuthToken(slot string) (string, error) {
+ return backend.GetStoredAuthToken(slot)
+}
+
+func (a *App) SetStoredAuthToken(slot, token string) error {
+ return backend.SetStoredAuthToken(slot, token)
+}
+
+func (a *App) ClearStoredAuthToken(slot string) error {
+ return backend.ClearStoredAuthToken(slot)
+}
+
func (a *App) IsExtractorInstalled() bool {
return backend.IsExtractorInstalled()
}
@@ -162,6 +175,10 @@ func (a *App) DownloadExtractor() error {
return backend.DownloadExtractor(nil)
}
+func (a *App) GetExtractorVersionStatus() backend.ExtractorVersionStatus {
+ return backend.GetExtractorVersionStatus()
+}
+
func (a *App) Quit() {
panic("quit")
}
@@ -192,6 +209,7 @@ type DownloadMediaWithMetadataRequest struct {
type DownloadMediaResponse struct {
Success bool `json:"success"`
+ Cancelled bool `json:"cancelled"`
Downloaded int `json:"downloaded"`
Skipped int `json:"skipped"`
Failed int `json:"failed"`
@@ -201,8 +219,9 @@ type DownloadMediaResponse struct {
func (a *App) DownloadMedia(req DownloadMediaRequest) (DownloadMediaResponse, error) {
if len(req.URLs) == 0 {
return DownloadMediaResponse{
- Success: false,
- Message: "No URLs provided",
+ Success: false,
+ Cancelled: false,
+ Message: "No URLs provided",
}, fmt.Errorf("no URLs provided")
}
@@ -219,6 +238,7 @@ func (a *App) DownloadMedia(req DownloadMediaRequest) (DownloadMediaResponse, er
if err != nil {
return DownloadMediaResponse{
Success: false,
+ Cancelled: false,
Downloaded: downloaded,
Skipped: 0,
Failed: failed,
@@ -228,6 +248,7 @@ func (a *App) DownloadMedia(req DownloadMediaRequest) (DownloadMediaResponse, er
return DownloadMediaResponse{
Success: true,
+ Cancelled: false,
Downloaded: downloaded,
Skipped: 0,
Failed: failed,
@@ -250,8 +271,9 @@ type DownloadItemStatus struct {
func (a *App) DownloadMediaWithMetadata(req DownloadMediaWithMetadataRequest) (DownloadMediaResponse, error) {
if len(req.Items) == 0 {
return DownloadMediaResponse{
- Success: false,
- Message: "No items provided",
+ Success: false,
+ Cancelled: false,
+ Message: "No items provided",
}, fmt.Errorf("no items provided")
}
@@ -286,6 +308,10 @@ func (a *App) DownloadMediaWithMetadata(req DownloadMediaWithMetadataRequest) (D
}
a.downloadCtx, a.downloadCancel = context.WithCancel(context.Background())
+ defer func() {
+ a.downloadCtx = nil
+ a.downloadCancel = nil
+ }()
progressCallback := func(current, total int) {
percent := 0
@@ -309,8 +335,20 @@ func (a *App) DownloadMediaWithMetadata(req DownloadMediaWithMetadataRequest) (D
downloaded, skipped, failed, err := backend.DownloadMediaWithMetadataProgressAndStatus(items, outputDir, req.Username, progressCallback, itemStatusCallback, a.downloadCtx, req.Proxy)
if err != nil {
+ if errors.Is(err, context.Canceled) {
+ return DownloadMediaResponse{
+ Success: false,
+ Cancelled: true,
+ Downloaded: downloaded,
+ Skipped: skipped,
+ Failed: failed,
+ Message: "Download stopped",
+ }, nil
+ }
+
return DownloadMediaResponse{
Success: false,
+ Cancelled: false,
Downloaded: downloaded,
Skipped: skipped,
Failed: failed,
@@ -318,10 +356,9 @@ func (a *App) DownloadMediaWithMetadata(req DownloadMediaWithMetadataRequest) (D
}, err
}
- a.downloadCancel = nil
-
return DownloadMediaResponse{
Success: true,
+ Cancelled: false,
Downloaded: downloaded,
Skipped: skipped,
Failed: failed,
@@ -358,6 +395,14 @@ func (a *App) GetAccountFromDB(id int64) (string, error) {
return acc.ResponseJSON, nil
}
+func (a *App) GetSavedAccountFromDB(username, mediaType string) (string, error) {
+ acc, err := backend.GetAccountByUsernameAndMediaType(username, mediaType)
+ if err != nil {
+ return "", err
+ }
+ return acc.ResponseJSON, nil
+}
+
func (a *App) DeleteAccountFromDB(id int64) error {
return backend.DeleteAccount(id)
}
@@ -382,6 +427,10 @@ func (a *App) GetAllGroups() ([]map[string]string, error) {
return backend.GetAllGroups()
}
+func (a *App) CheckFoldersExist(basePath string, usernames []string) map[string]bool {
+ return backend.CheckFoldersExist(basePath, usernames)
+}
+
func (a *App) IsFFmpegInstalled() bool {
return backend.IsFFmpegInstalled()
}
@@ -390,6 +439,10 @@ func (a *App) DownloadFFmpeg() error {
return backend.DownloadFFmpeg(nil)
}
+func (a *App) GetFFmpegVersionStatus() backend.DependencyVersionStatus {
+ return backend.GetFFmpegVersionStatus()
+}
+
func (a *App) IsExifToolInstalled() bool {
return backend.IsExifToolInstalled()
}
@@ -398,6 +451,10 @@ func (a *App) DownloadExifTool() error {
return backend.DownloadExifTool(nil)
}
+func (a *App) GetExifToolVersionStatus() backend.DependencyVersionStatus {
+ return backend.GetExifToolVersionStatus()
+}
+
type ConvertGIFsRequest struct {
FolderPath string `json:"folder_path"`
Quality string `json:"quality"`
diff --git a/backend/auth_token.go b/backend/auth_token.go
new file mode 100644
index 0000000..1cce1bb
--- /dev/null
+++ b/backend/auth_token.go
@@ -0,0 +1,70 @@
+package backend
+
+import (
+ "errors"
+ "fmt"
+ "strings"
+
+ "github.com/zalando/go-keyring"
+)
+
+const authTokenKeyringService = "TwitterXMediaBatchDownloader"
+
+func normalizeAuthTokenSlot(slot string) (string, error) {
+ switch strings.ToLower(strings.TrimSpace(slot)) {
+ case "public":
+ return "public", nil
+ case "private":
+ return "private", nil
+ default:
+ return "", fmt.Errorf("invalid auth token slot: %q", slot)
+ }
+}
+
+func authTokenKeyringUser(slot string) string {
+ return "auth-token-" + slot
+}
+
+func GetStoredAuthToken(slot string) (string, error) {
+ normalizedSlot, err := normalizeAuthTokenSlot(slot)
+ if err != nil {
+ return "", err
+ }
+
+ token, err := keyring.Get(authTokenKeyringService, authTokenKeyringUser(normalizedSlot))
+ if errors.Is(err, keyring.ErrNotFound) {
+ return "", nil
+ }
+ if err != nil {
+ return "", err
+ }
+
+ return token, nil
+}
+
+func SetStoredAuthToken(slot, token string) error {
+ normalizedSlot, err := normalizeAuthTokenSlot(slot)
+ if err != nil {
+ return err
+ }
+
+ if strings.TrimSpace(token) == "" {
+ return ClearStoredAuthToken(normalizedSlot)
+ }
+
+ return keyring.Set(authTokenKeyringService, authTokenKeyringUser(normalizedSlot), token)
+}
+
+func ClearStoredAuthToken(slot string) error {
+ normalizedSlot, err := normalizeAuthTokenSlot(slot)
+ if err != nil {
+ return err
+ }
+
+ err = keyring.Delete(authTokenKeyringService, authTokenKeyringUser(normalizedSlot))
+ if errors.Is(err, keyring.ErrNotFound) {
+ return nil
+ }
+
+ return err
+}
diff --git a/backend/config.go b/backend/config.go
index f6fc8f3..e28db0b 100644
--- a/backend/config.go
+++ b/backend/config.go
@@ -6,6 +6,8 @@ import (
"net/url"
"os"
"path/filepath"
+ "strings"
+ "sync/atomic"
"time"
)
@@ -20,14 +22,42 @@ func GetDefaultDownloadPath() string {
return filepath.Join(homeDir, "Pictures")
}
-func GetProxyURL(customProxy string) (*url.URL, error) {
+func parseProxyURLs(rawValue string, sourceName string) ([]*url.URL, error) {
+ if strings.TrimSpace(rawValue) == "" {
+ return nil, nil
+ }
+
+ parts := strings.Split(rawValue, ",")
+ proxyURLs := make([]*url.URL, 0, len(parts))
+
+ for _, part := range parts {
+ trimmed := strings.TrimSpace(part)
+ if trimmed == "" {
+ continue
+ }
+
+ proxyURL, err := url.Parse(trimmed)
+ if err != nil {
+ return nil, fmt.Errorf("invalid proxy URL in %s: %v", sourceName, err)
+ }
+ if proxyURL.Scheme == "" || proxyURL.Host == "" {
+ return nil, fmt.Errorf("invalid proxy URL in %s: %q", sourceName, trimmed)
+ }
+
+ proxyURLs = append(proxyURLs, proxyURL)
+ }
+
+ if len(proxyURLs) == 0 {
+ return nil, nil
+ }
+
+ return proxyURLs, nil
+}
+
+func GetProxyURLs(customProxy string) ([]*url.URL, error) {
if customProxy != "" {
- proxyURL, err := url.Parse(customProxy)
- if err != nil {
- return nil, fmt.Errorf("invalid custom proxy URL: %v", err)
- }
- return proxyURL, nil
+ return parseProxyURLs(customProxy, "custom proxy setting")
}
proxyEnv := os.Getenv("HTTPS_PROXY")
@@ -42,24 +72,43 @@ func GetProxyURL(customProxy string) (*url.URL, error) {
}
if proxyEnv != "" {
- proxyURL, err := url.Parse(proxyEnv)
- if err != nil {
- return nil, fmt.Errorf("invalid proxy URL from environment: %v", err)
- }
- return proxyURL, nil
+ return parseProxyURLs(proxyEnv, "proxy environment variable")
}
return nil, nil
}
+func GetProxyURL(customProxy string) (*url.URL, error) {
+ proxyURLs, err := GetProxyURLs(customProxy)
+ if err != nil || len(proxyURLs) == 0 {
+ return nil, err
+ }
+ return proxyURLs[0], nil
+}
+
+func buildProxySelector(proxyURLs []*url.URL) func(*http.Request) (*url.URL, error) {
+ if len(proxyURLs) == 0 {
+ return nil
+ }
+ if len(proxyURLs) == 1 {
+ return http.ProxyURL(proxyURLs[0])
+ }
+
+ var requestCount uint64
+ return func(*http.Request) (*url.URL, error) {
+ index := atomic.AddUint64(&requestCount, 1) - 1
+ return proxyURLs[index%uint64(len(proxyURLs))], nil
+ }
+}
+
func CreateHTTPClient(customProxy string, timeout time.Duration) (*http.Client, error) {
- proxyURL, err := GetProxyURL(customProxy)
+ proxyURLs, err := GetProxyURLs(customProxy)
if err != nil {
return nil, err
}
transport := &http.Transport{
- Proxy: http.ProxyURL(proxyURL),
+ Proxy: buildProxySelector(proxyURLs),
}
client := &http.Client{
diff --git a/backend/database.go b/backend/database.go
index ec2fcf0..d747475 100644
--- a/backend/database.go
+++ b/backend/database.go
@@ -42,6 +42,15 @@ type AccountListItem struct {
var db *sql.DB
+type accountMetricsPayload struct {
+ AccountInfo struct {
+ FollowersCount int `json:"followers_count"`
+ StatusesCount int `json:"statuses_count"`
+ } `json:"account_info"`
+ Followers int `json:"followers"`
+ Posts int `json:"posts"`
+}
+
func GetDBPath() string {
homeDir, err := os.UserHomeDir()
if err != nil {
@@ -82,6 +91,8 @@ func InitDB() error {
media_type TEXT DEFAULT 'all',
cursor TEXT DEFAULT '',
completed INTEGER DEFAULT 1,
+ followers_count INTEGER DEFAULT 0,
+ statuses_count INTEGER DEFAULT 0,
UNIQUE(username, media_type)
)
`)
@@ -94,9 +105,15 @@ func InitDB() error {
db.Exec("ALTER TABLE accounts ADD COLUMN media_type TEXT DEFAULT 'all'")
db.Exec("ALTER TABLE accounts ADD COLUMN cursor TEXT DEFAULT ''")
db.Exec("ALTER TABLE accounts ADD COLUMN completed INTEGER DEFAULT 1")
+ db.Exec("ALTER TABLE accounts ADD COLUMN followers_count INTEGER DEFAULT 0")
+ db.Exec("ALTER TABLE accounts ADD COLUMN statuses_count INTEGER DEFAULT 0")
db.Exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_username_media_type ON accounts(username, media_type)")
+ if err := backfillAccountMetrics(); err != nil {
+ return err
+ }
+
return nil
}
@@ -126,9 +143,11 @@ func SaveAccountWithStatus(username, name, profileImage string, totalMedia int,
completedInt = 1
}
+ followersCount, statusesCount := extractAccountMetrics(responseJSON)
+
_, err := db.Exec(`
- INSERT INTO accounts (username, name, profile_image, total_media, last_fetched, response_json, media_type, cursor, completed)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+ INSERT INTO accounts (username, name, profile_image, total_media, last_fetched, response_json, media_type, cursor, completed, followers_count, statuses_count)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(username, media_type) DO UPDATE SET
name = excluded.name,
profile_image = excluded.profile_image,
@@ -136,8 +155,10 @@ func SaveAccountWithStatus(username, name, profileImage string, totalMedia int,
last_fetched = excluded.last_fetched,
response_json = excluded.response_json,
cursor = excluded.cursor,
- completed = excluded.completed
- `, username, name, profileImage, totalMedia, time.Now(), responseJSON, mediaType, cursor, completedInt)
+ completed = excluded.completed,
+ followers_count = excluded.followers_count,
+ statuses_count = excluded.statuses_count
+ `, username, name, profileImage, totalMedia, time.Now(), responseJSON, mediaType, cursor, completedInt, followersCount, statusesCount)
return err
}
@@ -154,7 +175,8 @@ func GetAllAccounts() ([]AccountListItem, error) {
COALESCE(group_name, '') as group_name, COALESCE(group_color, '') as group_color,
COALESCE(media_type, 'all') as media_type,
COALESCE(cursor, '') as cursor, COALESCE(completed, 1) as completed,
- COALESCE(response_json, '') as response_json
+ COALESCE(followers_count, 0) as followers_count,
+ COALESCE(statuses_count, 0) as statuses_count
FROM accounts
ORDER BY group_name ASC, last_fetched DESC
`)
@@ -168,33 +190,79 @@ func GetAllAccounts() ([]AccountListItem, error) {
var acc AccountListItem
var lastFetched time.Time
var completedInt int
- var responseJSON string
- if err := rows.Scan(&acc.ID, &acc.Username, &acc.Name, &acc.ProfileImage, &acc.TotalMedia, &lastFetched, &acc.GroupName, &acc.GroupColor, &acc.MediaType, &acc.Cursor, &completedInt, &responseJSON); err != nil {
+ if err := rows.Scan(&acc.ID, &acc.Username, &acc.Name, &acc.ProfileImage, &acc.TotalMedia, &lastFetched, &acc.GroupName, &acc.GroupColor, &acc.MediaType, &acc.Cursor, &completedInt, &acc.FollowersCount, &acc.StatusesCount); err != nil {
continue
}
acc.LastFetched = lastFetched.Format("2006-01-02 15:04")
acc.Completed = completedInt == 1
- if responseJSON != "" {
- var parsed map[string]interface{}
- if err := json.Unmarshal([]byte(responseJSON), &parsed); err == nil {
- if accountInfo, ok := parsed["account_info"].(map[string]interface{}); ok {
- if followers, ok := accountInfo["followers_count"].(float64); ok {
- acc.FollowersCount = int(followers)
- }
- if statuses, ok := accountInfo["statuses_count"].(float64); ok {
- acc.StatusesCount = int(statuses)
- }
- }
- }
- }
-
accounts = append(accounts, acc)
}
return accounts, nil
}
+func extractAccountMetrics(responseJSON string) (followersCount int, statusesCount int) {
+ if responseJSON == "" {
+ return 0, 0
+ }
+
+ var payload accountMetricsPayload
+ if err := json.Unmarshal([]byte(responseJSON), &payload); err != nil {
+ return 0, 0
+ }
+
+ followersCount = payload.AccountInfo.FollowersCount
+ statusesCount = payload.AccountInfo.StatusesCount
+
+ if followersCount == 0 {
+ followersCount = payload.Followers
+ }
+ if statusesCount == 0 {
+ statusesCount = payload.Posts
+ }
+
+ return followersCount, statusesCount
+}
+
+func backfillAccountMetrics() error {
+ rows, err := db.Query(`
+ SELECT id, COALESCE(response_json, '') as response_json
+ FROM accounts
+ WHERE COALESCE(response_json, '') != ''
+ AND (COALESCE(followers_count, 0) = 0 OR COALESCE(statuses_count, 0) = 0)
+ `)
+ if err != nil {
+ return err
+ }
+ defer rows.Close()
+
+ stmt, err := db.Prepare(`
+ UPDATE accounts
+ SET followers_count = ?, statuses_count = ?
+ WHERE id = ?
+ `)
+ if err != nil {
+ return err
+ }
+ defer stmt.Close()
+
+ for rows.Next() {
+ var id int64
+ var responseJSON string
+ if err := rows.Scan(&id, &responseJSON); err != nil {
+ continue
+ }
+
+ followersCount, statusesCount := extractAccountMetrics(responseJSON)
+ if _, err := stmt.Exec(followersCount, statusesCount, id); err != nil {
+ return err
+ }
+ }
+
+ return rows.Err()
+}
+
func UpdateAccountGroup(id int64, groupName, groupColor string) error {
if db == nil {
if err := InitDB(); err != nil {
@@ -276,6 +344,41 @@ func GetAccountByID(id int64) (*AccountDB, error) {
return &acc, nil
}
+func GetAccountByUsernameAndMediaType(username, mediaType string) (*AccountDB, error) {
+ if db == nil {
+ if err := InitDB(); err != nil {
+ return nil, err
+ }
+ }
+
+ if mediaType == "" {
+ mediaType = "all"
+ }
+
+ var acc AccountDB
+ var lastFetched time.Time
+ var completedInt int
+ err := db.QueryRow(`
+ SELECT id, username, name, profile_image, total_media, last_fetched, response_json,
+ COALESCE(media_type, 'all') as media_type,
+ COALESCE(cursor, '') as cursor, COALESCE(completed, 1) as completed
+ FROM accounts
+ WHERE LOWER(username) = LOWER(?) AND COALESCE(media_type, 'all') = ?
+ `, username, mediaType).Scan(&acc.ID, &acc.Username, &acc.Name, &acc.ProfileImage, &acc.TotalMedia, &lastFetched, &acc.ResponseJSON, &acc.MediaType, &acc.Cursor, &completedInt)
+ if err != nil {
+ return nil, err
+ }
+
+ acc.LastFetched = lastFetched
+ acc.Completed = completedInt == 1
+
+ if converted, err := ConvertLegacyToNewFormat(acc.ResponseJSON); err == nil {
+ acc.ResponseJSON = converted
+ }
+
+ return &acc, nil
+}
+
func DeleteAccount(id int64) error {
if db == nil {
if err := InitDB(); err != nil {
diff --git a/backend/dependency_version.go b/backend/dependency_version.go
new file mode 100644
index 0000000..9097116
--- /dev/null
+++ b/backend/dependency_version.go
@@ -0,0 +1,7 @@
+package backend
+
+type DependencyVersionStatus struct {
+ Installed bool `json:"installed"`
+ InstalledVersion string `json:"installed_version,omitempty"`
+ LatestVersion string `json:"latest_version,omitempty"`
+}
diff --git a/backend/download.go b/backend/download.go
index f1a74d6..6d60ef8 100644
--- a/backend/download.go
+++ b/backend/download.go
@@ -175,6 +175,12 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str
}
var status string
+ markCompleted := func() {
+ completed := atomic.AddInt64(&completedCount, 1)
+ if progress != nil {
+ progress(int(completed), total)
+ }
+ }
if _, err := os.Stat(task.outputPath); err == nil {
status = "skipped"
@@ -183,6 +189,7 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str
itemStatus(task.item.TweetID, task.index, status)
}
atomic.AddInt64(&skippedCount, 1)
+ markCompleted()
continue
} else if task.item.Type == "text" {
@@ -214,10 +221,7 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str
itemStatus(task.item.TweetID, task.index, status)
}
- completed := atomic.AddInt64(&completedCount, 1)
- if progress != nil {
- progress(int(completed), total)
- }
+ markCompleted()
}
}()
}
diff --git a/backend/exiftool.go b/backend/exiftool.go
index 823be75..5a4b6e8 100644
--- a/backend/exiftool.go
+++ b/backend/exiftool.go
@@ -2,32 +2,56 @@ package backend
import (
"archive/zip"
- "crypto/sha256"
- "encoding/hex"
+ "encoding/json"
+ "encoding/xml"
"fmt"
+ "html"
"io"
"net/http"
"os"
"os/exec"
"path/filepath"
+ "regexp"
"runtime"
"strings"
+ "time"
)
const (
- exiftoolWindows64URL = "https://sourceforge.net/projects/exiftool/files/exiftool-13.43_64.zip/download"
-
- exiftoolWindows32URL = "https://sourceforge.net/projects/exiftool/files/exiftool-13.43_32.zip/download"
-
- exiftoolUnixURL = "https://sourceforge.net/projects/exiftool/files/Image-ExifTool-13.43.tar.gz/download"
+ exiftoolBestReleaseAPIURL = "https://sourceforge.net/projects/exiftool/best_release.json"
+ exiftoolRSSURL = "https://sourceforge.net/projects/exiftool/rss?path=/"
+ exiftoolFilesBaseURL = "https://sourceforge.net/projects/exiftool/files"
+ sourceForgeUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"
)
-const (
- exiftoolWindows64Hash = ""
- exiftoolWindows32Hash = ""
- exiftoolUnixHash = ""
+var (
+ exiftoolVersionPattern = regexp.MustCompile(`(?i)(?:exiftool|image-exiftool)-([0-9]+(?:\.[0-9]+)+)`)
+ sourceForgeMirrorMetaPattern = regexp.MustCompile(`(?i)url=(https?://downloads\.sourceforge\.net/project/[^"'<> ]+)`)
+ sourceForgeMirrorURLPattern = regexp.MustCompile(`https?://downloads\.sourceforge\.net/project/[^"'<> ]+`)
)
+type sourceForgeBestRelease struct {
+ Release sourceForgeBestReleaseFile `json:"release"`
+}
+
+type sourceForgeBestReleaseFile struct {
+ Filename string `json:"filename"`
+ URL string `json:"url"`
+}
+
+type sourceForgeRSS struct {
+ Channel sourceForgeRSSChannel `xml:"channel"`
+}
+
+type sourceForgeRSSChannel struct {
+ Items []sourceForgeRSSItem `xml:"item"`
+}
+
+type sourceForgeRSSItem struct {
+ Title string `xml:"title"`
+ Link string `xml:"link"`
+}
+
func GetExifToolPath() string {
homeDir, _ := os.UserHomeDir()
baseDir := filepath.Join(homeDir, ".twitterxmediabatchdownloader")
@@ -36,11 +60,9 @@ func GetExifToolPath() string {
case "windows":
return filepath.Join(baseDir, "exiftool.exe")
default:
-
pattern := filepath.Join(baseDir, "Image-ExifTool-*")
matches, err := filepath.Glob(pattern)
if err == nil && len(matches) > 0 {
-
exiftoolPath := filepath.Join(matches[0], "exiftool")
if _, err := os.Stat(exiftoolPath); err == nil {
return exiftoolPath
@@ -52,44 +74,74 @@ func GetExifToolPath() string {
}
func IsExifToolInstalled() bool {
- exiftoolPath := GetExifToolPath()
- if _, err := os.Stat(exiftoolPath); err == nil {
+ _, err := getInstalledExifToolVersion()
+ return err == nil
+}
- cmd := exec.Command(exiftoolPath, "-ver")
- hideWindow(cmd)
- if err := cmd.Run(); err == nil {
- return true
+func GetExifToolVersionStatus() DependencyVersionStatus {
+ status := DependencyVersionStatus{
+ Installed: IsExifToolInstalled(),
+ }
+
+ if status.Installed {
+ if installedVersion, err := getInstalledExifToolVersion(); err == nil {
+ status.InstalledVersion = installedVersion
}
}
- return false
+
+ if latestVersion, err := fetchLatestExifToolVersion(); err == nil {
+ status.LatestVersion = latestVersion
+ }
+
+ return status
}
-func calculateSHA256(filePath string) (string, error) {
- file, err := os.Open(filePath)
+func getInstalledExifToolVersion() (string, error) {
+ exiftoolPath := GetExifToolPath()
+ if _, err := os.Stat(exiftoolPath); err != nil {
+ return "", fmt.Errorf("exiftool not installed")
+ }
+
+ cmd := exec.Command(exiftoolPath, "-ver")
+ hideWindow(cmd)
+ output, err := cmd.Output()
if err != nil {
- return "", err
- }
- defer file.Close()
-
- hash := sha256.New()
- if _, err := io.Copy(hash, file); err != nil {
- return "", err
+ return "", fmt.Errorf("failed to determine exiftool version: %v", err)
}
- return hex.EncodeToString(hash.Sum(nil)), nil
+ version := normalizeExifToolVersion(string(output))
+ if version == "" {
+ return "", fmt.Errorf("failed to determine exiftool version")
+ }
+
+ return version, nil
}
-func verifyHash(filePath, expectedHash string) error {
- actualHash, err := calculateSHA256(filePath)
- if err != nil {
- return fmt.Errorf("failed to calculate hash: %v", err)
+func normalizeExifToolVersion(version string) string {
+ version = strings.TrimSpace(version)
+ if version == "" {
+ return ""
}
- if !strings.EqualFold(actualHash, expectedHash) {
- return fmt.Errorf("hash verification failed: expected %s, got %s", expectedHash, actualHash)
+ fields := strings.Fields(version)
+ if len(fields) > 0 {
+ version = fields[0]
}
- return nil
+ if strings.HasPrefix(strings.ToLower(version), "v") {
+ version = strings.TrimPrefix(strings.TrimPrefix(version, "v"), "V")
+ }
+
+ if version == "" {
+ return ""
+ }
+
+ first := version[0]
+ if first >= '0' && first <= '9' {
+ return "v" + version
+ }
+
+ return version
}
func is64Bit() bool {
@@ -97,24 +149,14 @@ func is64Bit() bool {
}
func DownloadExifTool(progressCallback func(downloaded, total int64)) error {
- var downloadURL string
- var expectedHash string
+ latestVersion, err := fetchLatestExifToolVersion()
+ if err != nil {
+ return err
+ }
- switch runtime.GOOS {
- case "windows":
- if is64Bit() {
- downloadURL = exiftoolWindows64URL
- expectedHash = exiftoolWindows64Hash
- } else {
- downloadURL = exiftoolWindows32URL
- expectedHash = exiftoolWindows32Hash
- }
- case "linux", "darwin":
-
- downloadURL = exiftoolUnixURL
- expectedHash = exiftoolUnixHash
- default:
- return fmt.Errorf("unsupported platform: %s", runtime.GOOS)
+ downloadURL, err := buildLatestExifToolDownloadURL(latestVersion)
+ if err != nil {
+ return err
}
tempFile, err := os.CreateTemp("", "exiftool-*")
@@ -123,47 +165,12 @@ func DownloadExifTool(progressCallback func(downloaded, total int64)) error {
}
tempPath := tempFile.Name()
defer os.Remove(tempPath)
- defer tempFile.Close()
-
- resp, err := http.Get(downloadURL)
- if err != nil {
- return fmt.Errorf("failed to download exiftool: %v", err)
- }
- defer resp.Body.Close()
-
- if resp.StatusCode != http.StatusOK {
- return fmt.Errorf("failed to download exiftool: status %d", resp.StatusCode)
+ if err := tempFile.Close(); err != nil {
+ return fmt.Errorf("failed to prepare temp file: %v", err)
}
- total := resp.ContentLength
- var downloaded int64
- buf := make([]byte, 32*1024)
-
- for {
- n, err := resp.Body.Read(buf)
- if n > 0 {
- _, writeErr := tempFile.Write(buf[:n])
- if writeErr != nil {
- return fmt.Errorf("failed to write temp file: %v", writeErr)
- }
- downloaded += int64(n)
- if progressCallback != nil {
- progressCallback(downloaded, total)
- }
- }
- if err == io.EOF {
- break
- }
- if err != nil {
- return fmt.Errorf("failed to download: %v", err)
- }
- }
- tempFile.Close()
-
- if expectedHash != "" {
- if err := verifyHash(tempPath, expectedHash); err != nil {
- return fmt.Errorf("hash verification failed: %v", err)
- }
+ if err := downloadSourceForgeFile(downloadURL, tempPath, progressCallback); err != nil {
+ return err
}
exiftoolPath := GetExifToolPath()
@@ -176,13 +183,254 @@ func DownloadExifTool(progressCallback func(downloaded, total int64)) error {
case "windows":
return extractExifToolFromZip(tempPath, exiftoolPath)
case "linux", "darwin":
-
return extractExifToolFromTarGz(tempPath, exiftoolPath)
}
return nil
}
+func fetchLatestExifToolVersion() (string, error) {
+ version, err := fetchLatestExifToolVersionFromBestRelease()
+ if err == nil {
+ return version, nil
+ }
+
+ fallbackVersion, fallbackErr := fetchLatestExifToolVersionFromRSS()
+ if fallbackErr == nil {
+ return fallbackVersion, nil
+ }
+
+ return "", fmt.Errorf("failed to fetch latest exiftool version: %v; fallback failed: %v", err, fallbackErr)
+}
+
+func fetchLatestExifToolVersionFromBestRelease() (string, error) {
+ client, err := CreateHTTPClient("", 30*time.Second)
+ if err != nil {
+ return "", fmt.Errorf("failed to configure exiftool release client: %v", err)
+ }
+
+ req, err := http.NewRequest(http.MethodGet, exiftoolBestReleaseAPIURL, nil)
+ if err != nil {
+ return "", fmt.Errorf("failed to create exiftool release request: %v", err)
+ }
+ req.Header.Set("Accept", "application/json")
+ req.Header.Set("User-Agent", sourceForgeUserAgent)
+
+ resp, err := client.Do(req)
+ if err != nil {
+ return "", fmt.Errorf("failed to fetch exiftool release info: %v", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ return "", fmt.Errorf("failed to fetch exiftool release info: status %d", resp.StatusCode)
+ }
+
+ var bestRelease sourceForgeBestRelease
+ if err := json.NewDecoder(resp.Body).Decode(&bestRelease); err != nil {
+ return "", fmt.Errorf("failed to decode exiftool release info: %v", err)
+ }
+
+ version := parseExifToolVersion(bestRelease.Release.Filename)
+ if version == "" {
+ version = parseExifToolVersion(bestRelease.Release.URL)
+ }
+ if version == "" {
+ return "", fmt.Errorf("failed to parse exiftool version from best_release.json")
+ }
+
+ return version, nil
+}
+
+func fetchLatestExifToolVersionFromRSS() (string, error) {
+ client, err := CreateHTTPClient("", 30*time.Second)
+ if err != nil {
+ return "", fmt.Errorf("failed to configure exiftool RSS client: %v", err)
+ }
+
+ req, err := http.NewRequest(http.MethodGet, exiftoolRSSURL, nil)
+ if err != nil {
+ return "", fmt.Errorf("failed to create exiftool RSS request: %v", err)
+ }
+ req.Header.Set("Accept", "application/rss+xml, application/xml, text/xml")
+ req.Header.Set("User-Agent", sourceForgeUserAgent)
+
+ resp, err := client.Do(req)
+ if err != nil {
+ return "", fmt.Errorf("failed to fetch exiftool RSS: %v", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ return "", fmt.Errorf("failed to fetch exiftool RSS: status %d", resp.StatusCode)
+ }
+
+ var feed sourceForgeRSS
+ if err := xml.NewDecoder(resp.Body).Decode(&feed); err != nil {
+ return "", fmt.Errorf("failed to decode exiftool RSS: %v", err)
+ }
+
+ for _, item := range feed.Channel.Items {
+ version := parseExifToolVersion(item.Title)
+ if version == "" {
+ version = parseExifToolVersion(item.Link)
+ }
+ if version != "" {
+ return version, nil
+ }
+ }
+
+ return "", fmt.Errorf("failed to find exiftool version in RSS feed")
+}
+
+func parseExifToolVersion(value string) string {
+ matches := exiftoolVersionPattern.FindStringSubmatch(value)
+ if len(matches) < 2 {
+ return ""
+ }
+
+ return normalizeExifToolVersion(matches[1])
+}
+
+func buildLatestExifToolDownloadURL(version string) (string, error) {
+ version = normalizeExifToolVersion(version)
+ if version == "" {
+ return "", fmt.Errorf("missing exiftool version")
+ }
+ version = strings.TrimPrefix(strings.TrimPrefix(version, "v"), "V")
+
+ switch runtime.GOOS {
+ case "windows":
+ if is64Bit() {
+ return fmt.Sprintf("%s/exiftool-%s_64.zip/download", exiftoolFilesBaseURL, version), nil
+ }
+ return fmt.Sprintf("%s/exiftool-%s_32.zip/download", exiftoolFilesBaseURL, version), nil
+ case "linux", "darwin":
+ return fmt.Sprintf("%s/Image-ExifTool-%s.tar.gz/download", exiftoolFilesBaseURL, version), nil
+ default:
+ return "", fmt.Errorf("unsupported platform: %s", runtime.GOOS)
+ }
+}
+
+func downloadSourceForgeFile(downloadURL, destinationPath string, progressCallback func(downloaded, total int64)) error {
+ client, err := CreateHTTPClient("", 10*time.Minute)
+ if err != nil {
+ return fmt.Errorf("failed to configure SourceForge download client: %v", err)
+ }
+
+ err = downloadSourceForgeURL(client, downloadURL, destinationPath, progressCallback, 0)
+ if err == nil {
+ return nil
+ }
+
+ if runtime.GOOS == "windows" {
+ if fallbackErr := downloadSourceForgeFileWithCurl(downloadURL, destinationPath); fallbackErr == nil {
+ return nil
+ } else {
+ return fmt.Errorf("%v; curl fallback failed: %v", err, fallbackErr)
+ }
+ }
+
+ return err
+}
+
+func downloadSourceForgeURL(client *http.Client, downloadURL, destinationPath string, progressCallback func(downloaded, total int64), depth int) error {
+ if depth > 4 {
+ return fmt.Errorf("failed to resolve SourceForge mirror URL")
+ }
+
+ req, err := http.NewRequest(http.MethodGet, downloadURL, nil)
+ if err != nil {
+ return fmt.Errorf("failed to create SourceForge download request: %v", err)
+ }
+ req.Header.Set("User-Agent", sourceForgeUserAgent)
+
+ resp, err := client.Do(req)
+ if err != nil {
+ return fmt.Errorf("failed to download exiftool: %v", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ return fmt.Errorf("failed to download exiftool: status %d", resp.StatusCode)
+ }
+
+ contentType := strings.ToLower(resp.Header.Get("Content-Type"))
+ if strings.Contains(contentType, "text/html") {
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return fmt.Errorf("failed to read SourceForge download page: %v", err)
+ }
+
+ resolvedURL := extractSourceForgeMirrorURL(string(body))
+ if resolvedURL == "" {
+ return fmt.Errorf("failed to resolve SourceForge mirror URL from download page")
+ }
+
+ return downloadSourceForgeURL(client, resolvedURL, destinationPath, progressCallback, depth+1)
+ }
+
+ destination, err := os.OpenFile(destinationPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+ if err != nil {
+ return fmt.Errorf("failed to create exiftool archive: %v", err)
+ }
+ defer destination.Close()
+
+ total := resp.ContentLength
+ var downloaded int64
+ buf := make([]byte, 32*1024)
+
+ for {
+ n, err := resp.Body.Read(buf)
+ if n > 0 {
+ if _, writeErr := destination.Write(buf[:n]); writeErr != nil {
+ return fmt.Errorf("failed to write exiftool archive: %v", writeErr)
+ }
+ downloaded += int64(n)
+ if progressCallback != nil {
+ progressCallback(downloaded, total)
+ }
+ }
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return fmt.Errorf("failed to download exiftool: %v", err)
+ }
+ }
+
+ return nil
+}
+
+func downloadSourceForgeFileWithCurl(downloadURL, destinationPath string) error {
+ curlPath, err := exec.LookPath("curl.exe")
+ if err != nil {
+ curlPath, err = exec.LookPath("curl")
+ if err != nil {
+ return fmt.Errorf("curl executable not found")
+ }
+ }
+
+ cmd := exec.Command(curlPath, "-L", "-o", destinationPath, downloadURL)
+ hideWindow(cmd)
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ return fmt.Errorf("failed to download via curl: %v, output: %s", err, string(output))
+ }
+
+ return nil
+}
+
+func extractSourceForgeMirrorURL(page string) string {
+ page = html.UnescapeString(page)
+
+ if matches := sourceForgeMirrorMetaPattern.FindStringSubmatch(page); len(matches) > 1 {
+ return strings.TrimSpace(matches[1])
+ }
+
+ return strings.TrimSpace(sourceForgeMirrorURLPattern.FindString(page))
+}
+
func extractExifToolFromZip(zipPath, destPath string) error {
r, err := zip.OpenReader(zipPath)
if err != nil {
@@ -222,9 +470,7 @@ func extractExifToolFromZip(zipPath, destPath string) error {
libDir := filepath.Join(filepath.Dir(destPath), "exiftool_files")
for _, f := range r.File {
-
if strings.Contains(f.Name, "exiftool_files") && !strings.HasSuffix(f.Name, "/") {
-
parts := strings.Split(f.Name, "exiftool_files/")
if len(parts) < 2 {
continue
@@ -266,7 +512,6 @@ func extractExifToolFromZip(zipPath, destPath string) error {
}
func extractExifToolFromTarGz(tarGzPath, destPath string) error {
-
baseDir := filepath.Dir(destPath)
cmd := exec.Command("tar", "-xzf", tarGzPath, "-C", baseDir)
@@ -282,22 +527,15 @@ func extractExifToolFromTarGz(tarGzPath, destPath string) error {
pattern := filepath.Join(baseDir, "Image-ExifTool-*")
matches, err := filepath.Glob(pattern)
if err == nil && len(matches) > 0 {
-
extractedDir = matches[0]
exiftoolScript = filepath.Join(extractedDir, "exiftool")
} else {
-
- extractedDir = filepath.Join(baseDir, "Image-ExifTool-13.43")
+ extractedDir = filepath.Join(baseDir, "exiftool")
exiftoolScript = filepath.Join(extractedDir, "exiftool")
}
if _, err := os.Stat(exiftoolScript); err != nil {
-
- extractedDir = filepath.Join(baseDir, "exiftool")
- exiftoolScript = filepath.Join(extractedDir, "exiftool")
- if _, err := os.Stat(exiftoolScript); err != nil {
- return fmt.Errorf("exiftool script not found in extracted archive (searched in: %s and %s)", pattern, extractedDir)
- }
+ return fmt.Errorf("exiftool script not found in extracted archive (searched in: %s and %s)", pattern, extractedDir)
}
if err := os.Chmod(exiftoolScript, 0755); err != nil {
diff --git a/backend/extractor.go b/backend/extractor.go
index 3950844..3fbca43 100644
--- a/backend/extractor.go
+++ b/backend/extractor.go
@@ -17,6 +17,19 @@ import (
)
const extractorReleaseAPIURL = "https://api.github.com/repos/afkarxyz/xtractor-binaries/releases/latest"
+const extractorReleasesAPIURL = "https://api.github.com/repos/afkarxyz/xtractor-binaries/releases"
+const extractorReleasePageSize = 20
+
+type ExtractorVersionStatus struct {
+ Installed bool `json:"installed"`
+ InstalledVersion string `json:"installed_version,omitempty"`
+ LatestVersion string `json:"latest_version,omitempty"`
+}
+
+type extractorVersionMetadata struct {
+ Version string `json:"version"`
+ AssetName string `json:"asset_name,omitempty"`
+}
type githubRelease struct {
TagName string `json:"tag_name"`
@@ -34,6 +47,25 @@ func IsExtractorInstalled() bool {
return err == nil && !info.IsDir()
}
+func GetExtractorVersionStatus() ExtractorVersionStatus {
+ status := ExtractorVersionStatus{
+ Installed: IsExtractorInstalled(),
+ }
+
+ assetName, assetErr := getExtractorAssetName()
+ if status.Installed && assetErr == nil {
+ if installedVersion, err := getInstalledExtractorVersion(assetName); err == nil {
+ status.InstalledVersion = installedVersion
+ }
+ }
+
+ if latestRelease, err := fetchLatestExtractorRelease(); err == nil {
+ status.LatestVersion = normalizeExtractorVersion(latestRelease.TagName)
+ }
+
+ return status
+}
+
func DownloadExtractor(progressCallback func(downloaded, total int64)) error {
assetName, err := getExtractorAssetName()
if err != nil {
@@ -82,6 +114,10 @@ func DownloadExtractor(progressCallback func(downloaded, total int64)) error {
}
_ = removeLegacyExtractorBinary()
+ _ = writeExtractorVersionMetadata(extractorVersionMetadata{
+ Version: normalizeExtractorVersion(release.TagName),
+ AssetName: asset.Name,
+ })
return nil
}
@@ -157,12 +193,10 @@ func fetchLatestExtractorRelease() (*githubRelease, error) {
return nil, fmt.Errorf("failed to configure xtractor network client: %v", err)
}
- req, err := http.NewRequest(http.MethodGet, extractorReleaseAPIURL, nil)
+ req, err := newExtractorGitHubRequest(extractorReleaseAPIURL)
if err != nil {
- return nil, fmt.Errorf("failed to create xtractor release request: %v", err)
+ return nil, err
}
- req.Header.Set("Accept", "application/vnd.github+json")
- req.Header.Set("User-Agent", "Twitter-X-Media-Batch-Downloader")
resp, err := client.Do(req)
if err != nil {
@@ -182,6 +216,46 @@ func fetchLatestExtractorRelease() (*githubRelease, error) {
return &release, nil
}
+func fetchExtractorReleases(limit int) ([]githubRelease, error) {
+ client, err := CreateHTTPClient("", 60*time.Second)
+ if err != nil {
+ return nil, fmt.Errorf("failed to configure xtractor release client: %v", err)
+ }
+
+ req, err := newExtractorGitHubRequest(fmt.Sprintf("%s?per_page=%d", extractorReleasesAPIURL, limit))
+ if err != nil {
+ return nil, err
+ }
+
+ resp, err := client.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to fetch xtractor releases: %v", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("failed to fetch xtractor releases: status %d", resp.StatusCode)
+ }
+
+ var releases []githubRelease
+ if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil {
+ return nil, fmt.Errorf("failed to decode xtractor releases: %v", err)
+ }
+
+ return releases, nil
+}
+
+func newExtractorGitHubRequest(url string) (*http.Request, error) {
+ req, err := http.NewRequest(http.MethodGet, url, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create xtractor GitHub request: %v", err)
+ }
+
+ req.Header.Set("Accept", "application/vnd.github+json")
+ req.Header.Set("User-Agent", "Twitter-X-Media-Batch-Downloader")
+ return req, nil
+}
+
func findExtractorAsset(assets []githubReleaseAsset, assetName string) (*githubReleaseAsset, error) {
for i := range assets {
if assets[i].Name == assetName {
@@ -348,6 +422,177 @@ func findExtractorBinaryEntry(entries []*zip.File) (*zip.File, error) {
return nil, fmt.Errorf("xtractor binary not found in archive")
}
+func getInstalledExtractorVersion(assetName string) (string, error) {
+ metadata, err := readExtractorVersionMetadata()
+ if err == nil {
+ version := normalizeExtractorVersion(metadata.Version)
+ if version != "" && (metadata.AssetName == "" || strings.EqualFold(metadata.AssetName, assetName)) {
+ return version, nil
+ }
+ }
+
+ version, err := resolveInstalledExtractorVersionByHash(assetName)
+ if err != nil {
+ return "", err
+ }
+
+ version = normalizeExtractorVersion(version)
+ if version != "" {
+ _ = writeExtractorVersionMetadata(extractorVersionMetadata{
+ Version: version,
+ AssetName: assetName,
+ })
+ }
+
+ return version, nil
+}
+
+func resolveInstalledExtractorVersionByHash(assetName string) (string, error) {
+ exePath, err := requireExtractorPath()
+ if err != nil {
+ return "", err
+ }
+
+ installedHash, err := hashFileSHA256(exePath)
+ if err != nil {
+ return "", fmt.Errorf("failed to hash installed xtractor: %v", err)
+ }
+
+ releases, err := fetchExtractorReleases(extractorReleasePageSize)
+ if err != nil {
+ return "", err
+ }
+
+ for _, release := range releases {
+ asset, err := findExtractorAsset(release.Assets, assetName)
+ if err != nil {
+ continue
+ }
+
+ matches, err := archiveMatchesExtractorHash(asset.BrowserDownloadURL, installedHash)
+ if err != nil {
+ continue
+ }
+
+ if matches {
+ return release.TagName, nil
+ }
+ }
+
+ return "", nil
+}
+
+func archiveMatchesExtractorHash(downloadURL, installedHash string) (bool, error) {
+ tempFile, err := os.CreateTemp("", "xtractor-version-*.zip")
+ if err != nil {
+ return false, fmt.Errorf("failed to create temp archive for version check: %v", err)
+ }
+
+ tempPath := tempFile.Name()
+ defer os.Remove(tempPath)
+ defer tempFile.Close()
+
+ if err := downloadExtractorArchive(downloadURL, tempFile, nil); err != nil {
+ return false, err
+ }
+
+ if err := tempFile.Close(); err != nil {
+ return false, fmt.Errorf("failed to finalize version archive: %v", err)
+ }
+
+ archiveHash, err := hashExtractorBinaryInArchive(tempPath)
+ if err != nil {
+ return false, err
+ }
+
+ return archiveHash == installedHash, nil
+}
+
+func hashExtractorBinaryInArchive(zipPath string) (string, error) {
+ reader, err := zip.OpenReader(zipPath)
+ if err != nil {
+ return "", fmt.Errorf("failed to open xtractor archive for version check: %v", err)
+ }
+ defer reader.Close()
+
+ entry, err := findExtractorBinaryEntry(reader.File)
+ if err != nil {
+ return "", err
+ }
+
+ in, err := entry.Open()
+ if err != nil {
+ return "", fmt.Errorf("failed to open xtractor binary from archive for version check: %v", err)
+ }
+ defer in.Close()
+
+ return hashReaderSHA256(in)
+}
+
+func hashFileSHA256(filePath string) (string, error) {
+ file, err := os.Open(filePath)
+ if err != nil {
+ return "", fmt.Errorf("failed to open xtractor binary: %v", err)
+ }
+ defer file.Close()
+
+ return hashReaderSHA256(file)
+}
+
+func hashReaderSHA256(reader io.Reader) (string, error) {
+ hash := sha256.New()
+ if _, err := io.Copy(hash, reader); err != nil {
+ return "", err
+ }
+
+ return hex.EncodeToString(hash.Sum(nil)), nil
+}
+
+func getExtractorVersionMetadataPath() string {
+ return filepath.Join(filepath.Dir(getExtractorPath()), "xtractor-version.json")
+}
+
+func readExtractorVersionMetadata() (extractorVersionMetadata, error) {
+ data, err := os.ReadFile(getExtractorVersionMetadataPath())
+ if err != nil {
+ return extractorVersionMetadata{}, err
+ }
+
+ var metadata extractorVersionMetadata
+ if err := json.Unmarshal(data, &metadata); err != nil {
+ return extractorVersionMetadata{}, err
+ }
+
+ return metadata, nil
+}
+
+func writeExtractorVersionMetadata(metadata extractorVersionMetadata) error {
+ metadata.Version = normalizeExtractorVersion(metadata.Version)
+ if metadata.Version == "" {
+ return nil
+ }
+
+ path := getExtractorVersionMetadataPath()
+ if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+ return fmt.Errorf("failed to create xtractor metadata directory: %v", err)
+ }
+
+ data, err := json.Marshal(metadata)
+ if err != nil {
+ return fmt.Errorf("failed to encode xtractor metadata: %v", err)
+ }
+
+ if err := os.WriteFile(path, data, 0644); err != nil {
+ return fmt.Errorf("failed to write xtractor metadata: %v", err)
+ }
+
+ return nil
+}
+
+func normalizeExtractorVersion(version string) string {
+ return strings.TrimSpace(version)
+}
+
func getLegacyExtractorPath() string {
homeDir, _ := os.UserHomeDir()
baseDir := filepath.Join(homeDir, ".twitterxmediabatchdownloader")
diff --git a/backend/ffmpeg.go b/backend/ffmpeg.go
index 36f8b10..2a4608b 100644
--- a/backend/ffmpeg.go
+++ b/backend/ffmpeg.go
@@ -1,26 +1,41 @@
package backend
import (
- "archive/tar"
"archive/zip"
+ "encoding/json"
"fmt"
"io"
"net/http"
+ neturl "net/url"
"os"
"os/exec"
"path/filepath"
+ "regexp"
"runtime"
"strings"
-
- "github.com/ulikunitz/xz"
+ "time"
)
const (
- ffmpegWindowsURL = "https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-win64-gpl.zip"
- ffmpegLinuxURL = "https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz"
- ffmpegMacOSURL = "https://evermeet.cx/ffmpeg/getrelease/ffmpeg/zip"
+ ffmpegReleaseAPIURL = "https://api.github.com/repos/afkarxyz/ffmpeg-binaries/releases/latest"
+ ffmpegLatestReleaseURL = "https://github.com/afkarxyz/ffmpeg-binaries/releases/latest"
+ ffmpegLatestDownloadBaseURL = "https://github.com/afkarxyz/ffmpeg-binaries/releases/latest/download"
)
+var ffmpegVersionPattern = regexp.MustCompile(`^[0-9]+(?:\.[0-9]+)+`)
+
+type ffmpegVersionMetadata struct {
+ Version string `json:"version"`
+ AssetName string `json:"asset_name,omitempty"`
+}
+
+type ffmpegResolvedRelease struct {
+ Version string
+ AssetURL string
+ AssetName string
+ Digest string
+}
+
func GetFFmpegPath() string {
homeDir, _ := os.UserHomeDir()
baseDir := filepath.Join(homeDir, ".twitterxmediabatchdownloader")
@@ -35,27 +50,45 @@ func GetFFmpegPath() string {
func IsFFmpegInstalled() bool {
ffmpegPath := GetFFmpegPath()
- if _, err := os.Stat(ffmpegPath); err == nil {
- return true
+ if _, err := os.Stat(ffmpegPath); err != nil {
+ return false
}
- return false
+
+ cmd := exec.Command(ffmpegPath, "-version")
+ hideWindow(cmd)
+ return cmd.Run() == nil
+}
+
+func GetFFmpegVersionStatus() DependencyVersionStatus {
+ status := DependencyVersionStatus{
+ Installed: IsFFmpegInstalled(),
+ }
+
+ if status.Installed {
+ if installedVersion, err := getInstalledFFmpegVersion(); err == nil {
+ status.InstalledVersion = installedVersion
+ }
+ }
+
+ if latestVersion, err := fetchLatestFFmpegVersion(); err == nil {
+ status.LatestVersion = latestVersion
+ }
+
+ return status
}
func DownloadFFmpeg(progressCallback func(downloaded, total int64)) error {
- var downloadURL string
-
- switch runtime.GOOS {
- case "windows":
- downloadURL = ffmpegWindowsURL
- case "linux":
- downloadURL = ffmpegLinuxURL
- case "darwin":
- downloadURL = ffmpegMacOSURL
- default:
- return fmt.Errorf("unsupported platform: %s", runtime.GOOS)
+ assetName, err := getFFmpegAssetName()
+ if err != nil {
+ return err
}
- tempFile, err := os.CreateTemp("", "ffmpeg-*")
+ releaseInfo, err := resolveLatestFFmpegRelease(assetName)
+ if err != nil {
+ return err
+ }
+
+ tempFile, err := os.CreateTemp("", "ffmpeg-*.zip")
if err != nil {
return fmt.Errorf("failed to create temp file: %v", err)
}
@@ -63,7 +96,227 @@ func DownloadFFmpeg(progressCallback func(downloaded, total int64)) error {
defer os.Remove(tempPath)
defer tempFile.Close()
- resp, err := http.Get(downloadURL)
+ if err := downloadFFmpegArchive(releaseInfo.AssetURL, tempFile, progressCallback); err != nil {
+ return err
+ }
+
+ if err := tempFile.Close(); err != nil {
+ return fmt.Errorf("failed to finalize ffmpeg archive: %v", err)
+ }
+
+ if releaseInfo.Digest != "" {
+ if err := verifyDownloadedDigest(tempPath, releaseInfo.Digest); err != nil {
+ return err
+ }
+ }
+
+ ffmpegPath := GetFFmpegPath()
+ baseDir := filepath.Dir(ffmpegPath)
+ if err := os.MkdirAll(baseDir, 0755); err != nil {
+ return fmt.Errorf("failed to create ffmpeg directory: %v", err)
+ }
+
+ if err := extractFFmpegFromZip(tempPath, ffmpegPath); err != nil {
+ return err
+ }
+
+ _ = writeFFmpegVersionMetadata(ffmpegVersionMetadata{
+ Version: releaseInfo.Version,
+ AssetName: releaseInfo.AssetName,
+ })
+
+ return nil
+}
+
+func getFFmpegAssetName() (string, error) {
+ switch runtime.GOOS {
+ case "windows":
+ return "ffmpeg-windows.zip", nil
+ case "linux":
+ switch runtime.GOARCH {
+ case "amd64":
+ return "ffmpeg-linux-amd64.zip", nil
+ case "arm64":
+ return "ffmpeg-linux-arm64v8.zip", nil
+ }
+ case "darwin":
+ switch runtime.GOARCH {
+ case "amd64":
+ return "ffmpeg-macos-amd64.zip", nil
+ case "arm64":
+ return "ffmpeg-macos-arm64.zip", nil
+ }
+ }
+
+ return "", fmt.Errorf("unsupported platform for ffmpeg binary: %s/%s", runtime.GOOS, runtime.GOARCH)
+}
+
+func resolveLatestFFmpegRelease(assetName string) (ffmpegResolvedRelease, error) {
+ release, apiErr := fetchLatestFFmpegRelease()
+ if apiErr == nil {
+ asset, assetErr := findFFmpegAsset(release.Assets, assetName)
+ if assetErr == nil {
+ return ffmpegResolvedRelease{
+ Version: normalizeFFmpegVersion(release.TagName),
+ AssetURL: asset.BrowserDownloadURL,
+ AssetName: asset.Name,
+ Digest: asset.Digest,
+ }, nil
+ }
+ apiErr = assetErr
+ }
+
+ fallbackVersion, fallbackErr := fetchLatestFFmpegVersionFromRedirect()
+ if fallbackErr != nil {
+ return ffmpegResolvedRelease{}, fmt.Errorf("failed to resolve latest ffmpeg release: %v; fallback failed: %v", apiErr, fallbackErr)
+ }
+
+ return ffmpegResolvedRelease{
+ Version: fallbackVersion,
+ AssetURL: buildLatestFFmpegDownloadURL(assetName),
+ AssetName: assetName,
+ }, nil
+}
+
+func fetchLatestFFmpegVersion() (string, error) {
+ release, err := fetchLatestFFmpegRelease()
+ if err == nil {
+ version := normalizeFFmpegVersion(release.TagName)
+ if version != "" {
+ return version, nil
+ }
+ }
+
+ fallbackVersion, fallbackErr := fetchLatestFFmpegVersionFromRedirect()
+ if fallbackErr == nil {
+ return fallbackVersion, nil
+ }
+
+ if err != nil {
+ return "", fmt.Errorf("failed to fetch latest ffmpeg version: %v; fallback failed: %v", err, fallbackErr)
+ }
+
+ return "", fmt.Errorf("failed to determine latest ffmpeg version")
+}
+
+func fetchLatestFFmpegRelease() (*githubRelease, error) {
+ client, err := CreateHTTPClient("", 30*time.Second)
+ if err != nil {
+ return nil, fmt.Errorf("failed to configure ffmpeg network client: %v", err)
+ }
+
+ req, err := newFFmpegGitHubRequest(ffmpegReleaseAPIURL)
+ if err != nil {
+ return nil, err
+ }
+
+ resp, err := client.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to fetch ffmpeg release info: %v", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("failed to fetch ffmpeg release info: status %d", resp.StatusCode)
+ }
+
+ var release githubRelease
+ if err := json.NewDecoder(resp.Body).Decode(&release); err != nil {
+ return nil, fmt.Errorf("failed to decode ffmpeg release info: %v", err)
+ }
+
+ return &release, nil
+}
+
+func newFFmpegGitHubRequest(url string) (*http.Request, error) {
+ req, err := http.NewRequest(http.MethodGet, url, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create ffmpeg GitHub request: %v", err)
+ }
+
+ req.Header.Set("Accept", "application/vnd.github+json")
+ req.Header.Set("User-Agent", "Twitter-X-Media-Batch-Downloader")
+ return req, nil
+}
+
+func findFFmpegAsset(assets []githubReleaseAsset, assetName string) (*githubReleaseAsset, error) {
+ for i := range assets {
+ if assets[i].Name == assetName {
+ return &assets[i], nil
+ }
+ }
+
+ return nil, fmt.Errorf("ffmpeg asset %q not found in latest release", assetName)
+}
+
+func fetchLatestFFmpegVersionFromRedirect() (string, error) {
+ client, err := CreateHTTPClient("", 30*time.Second)
+ if err != nil {
+ return "", fmt.Errorf("failed to configure ffmpeg redirect client: %v", err)
+ }
+
+ client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+ return http.ErrUseLastResponse
+ }
+
+ req, err := http.NewRequest(http.MethodGet, ffmpegLatestReleaseURL, nil)
+ if err != nil {
+ return "", fmt.Errorf("failed to create ffmpeg redirect request: %v", err)
+ }
+ req.Header.Set("User-Agent", "Twitter-X-Media-Batch-Downloader")
+
+ resp, err := client.Do(req)
+ if err != nil {
+ return "", fmt.Errorf("failed to resolve latest ffmpeg redirect: %v", err)
+ }
+ defer resp.Body.Close()
+
+ location := resp.Header.Get("Location")
+ if location == "" && resp.Request != nil && resp.Request.URL != nil {
+ location = resp.Request.URL.String()
+ }
+
+ version := parseFFmpegVersionFromReleaseURL(location)
+ if version == "" {
+ return "", fmt.Errorf("failed to parse ffmpeg version from redirect URL")
+ }
+
+ return version, nil
+}
+
+func parseFFmpegVersionFromReleaseURL(releaseURL string) string {
+ parsed, err := neturl.Parse(strings.TrimSpace(releaseURL))
+ if err != nil {
+ return ""
+ }
+
+ segments := strings.Split(strings.Trim(parsed.Path, "/"), "/")
+ for index, segment := range segments {
+ if (segment == "tag" || segment == "download") && index+1 < len(segments) {
+ return normalizeFFmpegVersion(segments[index+1])
+ }
+ }
+
+ return ""
+}
+
+func buildLatestFFmpegDownloadURL(assetName string) string {
+ return ffmpegLatestDownloadBaseURL + "/" + assetName
+}
+
+func downloadFFmpegArchive(downloadURL string, destination *os.File, progressCallback func(downloaded, total int64)) error {
+ client, err := CreateHTTPClient("", 10*time.Minute)
+ if err != nil {
+ return fmt.Errorf("failed to configure ffmpeg download client: %v", err)
+ }
+
+ req, err := http.NewRequest(http.MethodGet, downloadURL, nil)
+ if err != nil {
+ return fmt.Errorf("failed to create ffmpeg download request: %v", err)
+ }
+ req.Header.Set("User-Agent", "Twitter-X-Media-Batch-Downloader")
+
+ resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to download ffmpeg: %v", err)
}
@@ -80,9 +333,8 @@ func DownloadFFmpeg(progressCallback func(downloaded, total int64)) error {
for {
n, err := resp.Body.Read(buf)
if n > 0 {
- _, writeErr := tempFile.Write(buf[:n])
- if writeErr != nil {
- return fmt.Errorf("failed to write temp file: %v", writeErr)
+ if _, writeErr := destination.Write(buf[:n]); writeErr != nil {
+ return fmt.Errorf("failed to write ffmpeg archive: %v", writeErr)
}
downloaded += int64(n)
if progressCallback != nil {
@@ -93,28 +345,14 @@ func DownloadFFmpeg(progressCallback func(downloaded, total int64)) error {
break
}
if err != nil {
- return fmt.Errorf("failed to download: %v", err)
+ return fmt.Errorf("failed to download ffmpeg: %v", err)
}
}
- tempFile.Close()
-
- ffmpegPath := GetFFmpegPath()
- baseDir := filepath.Dir(ffmpegPath)
- if err := os.MkdirAll(baseDir, 0755); err != nil {
- return fmt.Errorf("failed to create directory: %v", err)
- }
-
- switch runtime.GOOS {
- case "windows", "darwin":
- return extractFromZip(tempPath, ffmpegPath)
- case "linux":
- return extractFromTarXz(tempPath, ffmpegPath)
- }
return nil
}
-func extractFromZip(zipPath, destPath string) error {
+func extractFFmpegFromZip(zipPath, destPath string) error {
r, err := zip.OpenReader(zipPath)
if err != nil {
return fmt.Errorf("failed to open zip: %v", err)
@@ -122,77 +360,174 @@ func extractFromZip(zipPath, destPath string) error {
defer r.Close()
for _, f := range r.File {
-
name := filepath.Base(f.Name)
- if name == "ffmpeg" || name == "ffmpeg.exe" {
- rc, err := f.Open()
- if err != nil {
- return fmt.Errorf("failed to open file in zip: %v", err)
- }
- defer rc.Close()
-
- out, err := os.Create(destPath)
- if err != nil {
- return fmt.Errorf("failed to create output file: %v", err)
- }
- defer out.Close()
-
- if _, err := io.Copy(out, rc); err != nil {
- return fmt.Errorf("failed to extract file: %v", err)
- }
-
- if runtime.GOOS != "windows" {
- os.Chmod(destPath, 0755)
- }
-
- return nil
+ if name != "ffmpeg" && name != "ffmpeg.exe" {
+ continue
}
+
+ rc, err := f.Open()
+ if err != nil {
+ return fmt.Errorf("failed to open file in zip: %v", err)
+ }
+ defer rc.Close()
+
+ out, err := os.Create(destPath)
+ if err != nil {
+ return fmt.Errorf("failed to create output file: %v", err)
+ }
+ defer out.Close()
+
+ if _, err := io.Copy(out, rc); err != nil {
+ return fmt.Errorf("failed to extract file: %v", err)
+ }
+
+ if runtime.GOOS != "windows" {
+ if err := os.Chmod(destPath, 0755); err != nil {
+ return fmt.Errorf("failed to make ffmpeg executable: %v", err)
+ }
+ }
+
+ return nil
}
return fmt.Errorf("ffmpeg binary not found in archive")
}
-func extractFromTarXz(tarXzPath, destPath string) error {
- file, err := os.Open(tarXzPath)
+func getInstalledFFmpegVersion() (string, error) {
+ ffmpegPath := GetFFmpegPath()
+ if _, err := os.Stat(ffmpegPath); err != nil {
+ return "", fmt.Errorf("ffmpeg not installed")
+ }
+
+ cmd := exec.Command(ffmpegPath, "-version")
+ hideWindow(cmd)
+ output, err := cmd.Output()
+ if err == nil {
+ version := normalizeFFmpegVersionOutput(string(output))
+ if version != "" {
+ return version, nil
+ }
+ }
+
+ metadata, metadataErr := readFFmpegVersionMetadata()
+ if metadataErr == nil {
+ version := normalizeFFmpegVersion(metadata.Version)
+ if version != "" {
+ return version, nil
+ }
+ }
+
if err != nil {
- return fmt.Errorf("failed to open tar.xz: %v", err)
+ return "", fmt.Errorf("failed to determine ffmpeg version: %v", err)
}
- defer file.Close()
- xzReader, err := xz.NewReader(file)
+ return "", fmt.Errorf("failed to determine ffmpeg version")
+}
+
+func normalizeFFmpegVersionOutput(output string) string {
+ line := strings.TrimSpace(output)
+ if line == "" {
+ return ""
+ }
+
+ line = strings.Split(line, "\n")[0]
+ line = strings.TrimSpace(line)
+ lowerLine := strings.ToLower(line)
+ const prefix = "ffmpeg version "
+ if strings.HasPrefix(lowerLine, prefix) {
+ line = strings.TrimSpace(line[len(prefix):])
+ }
+
+ fields := strings.Fields(line)
+ if len(fields) > 0 {
+ line = fields[0]
+ }
+
+ return normalizeFFmpegVersion(line)
+}
+
+func getFFmpegVersionMetadataPath() string {
+ return filepath.Join(filepath.Dir(GetFFmpegPath()), "ffmpeg-version.json")
+}
+
+func readFFmpegVersionMetadata() (ffmpegVersionMetadata, error) {
+ data, err := os.ReadFile(getFFmpegVersionMetadataPath())
if err != nil {
- return fmt.Errorf("failed to create xz reader: %v", err)
+ return ffmpegVersionMetadata{}, err
}
- tarReader := tar.NewReader(xzReader)
+ var metadata ffmpegVersionMetadata
+ if err := json.Unmarshal(data, &metadata); err != nil {
+ return ffmpegVersionMetadata{}, err
+ }
- for {
- header, err := tarReader.Next()
- if err == io.EOF {
- break
- }
- if err != nil {
- return fmt.Errorf("failed to read tar: %v", err)
- }
+ return metadata, nil
+}
- name := filepath.Base(header.Name)
- if name == "ffmpeg" && header.Typeflag == tar.TypeReg {
- out, err := os.Create(destPath)
- if err != nil {
- return fmt.Errorf("failed to create output file: %v", err)
- }
- defer out.Close()
+func writeFFmpegVersionMetadata(metadata ffmpegVersionMetadata) error {
+ metadata.Version = normalizeFFmpegVersion(metadata.Version)
+ if metadata.Version == "" {
+ return nil
+ }
- if _, err := io.Copy(out, tarReader); err != nil {
- return fmt.Errorf("failed to extract file: %v", err)
- }
+ path := getFFmpegVersionMetadataPath()
+ if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+ return fmt.Errorf("failed to create ffmpeg metadata directory: %v", err)
+ }
- os.Chmod(destPath, 0755)
- return nil
+ data, err := json.Marshal(metadata)
+ if err != nil {
+ return fmt.Errorf("failed to encode ffmpeg metadata: %v", err)
+ }
+
+ if err := os.WriteFile(path, data, 0644); err != nil {
+ return fmt.Errorf("failed to write ffmpeg metadata: %v", err)
+ }
+
+ return nil
+}
+
+func normalizeFFmpegVersion(version string) string {
+ version = strings.TrimSpace(version)
+ if version == "" {
+ return ""
+ }
+
+ lowerVersion := strings.ToLower(version)
+ if strings.HasPrefix(lowerVersion, "ffmpeg version ") {
+ version = strings.TrimSpace(version[len("ffmpeg version "):])
+ }
+
+ fields := strings.Fields(version)
+ if len(fields) > 0 {
+ version = fields[0]
+ }
+
+ if strings.HasPrefix(strings.ToLower(version), "n") && len(version) > 1 {
+ next := version[1]
+ if next >= '0' && next <= '9' {
+ version = version[1:]
}
}
- return fmt.Errorf("ffmpeg binary not found in archive")
+ if strings.HasPrefix(strings.ToLower(version), "v") {
+ version = strings.TrimPrefix(strings.TrimPrefix(version, "v"), "V")
+ }
+
+ if version == "" {
+ return ""
+ }
+
+ if numericVersion := ffmpegVersionPattern.FindString(version); numericVersion != "" {
+ version = numericVersion
+ }
+
+ first := version[0]
+ if first >= '0' && first <= '9' {
+ return "v" + version
+ }
+
+ return version
}
func ConvertMP4ToGIF(inputPath, outputPath, quality, resolution string) error {
@@ -205,7 +540,6 @@ func ConvertMP4ToGIF(inputPath, outputPath, quality, resolution string) error {
var args []string
if quality == "fast" {
-
var scaleFilter string
switch resolution {
case "high":
@@ -235,7 +569,6 @@ func ConvertMP4ToGIF(inputPath, outputPath, quality, resolution string) error {
}
}
} else {
-
var filter string
switch resolution {
diff --git a/backend/folder.go b/backend/folder.go
index f8608d6..2bf4a01 100644
--- a/backend/folder.go
+++ b/backend/folder.go
@@ -61,6 +61,25 @@ func CheckFolderExists(basePath, username string) bool {
return info.IsDir()
}
+func CheckFoldersExist(basePath string, usernames []string) map[string]bool {
+ results := make(map[string]bool, len(usernames))
+ if basePath == "" {
+ return results
+ }
+
+ checked := make(map[string]bool, len(usernames))
+ for _, username := range usernames {
+ if _, alreadyChecked := checked[username]; alreadyChecked {
+ continue
+ }
+ exists := CheckFolderExists(basePath, username)
+ checked[username] = exists
+ results[username] = exists
+ }
+
+ return results
+}
+
func CheckGifsFolderExists(basePath, username string) bool {
gifsPath := filepath.Join(basePath, username, "gifs")
info, err := os.Stat(gifsPath)
diff --git a/backend/twitter.go b/backend/twitter.go
index aa2a352..64f53b3 100644
--- a/backend/twitter.go
+++ b/backend/twitter.go
@@ -1,6 +1,7 @@
package backend
import (
+ "bytes"
"encoding/json"
"fmt"
"net/url"
@@ -9,8 +10,53 @@ import (
"path/filepath"
"runtime"
"strings"
+ "sync"
)
+var activeExtractorCommands = struct {
+ sync.Mutex
+ byPID map[int]*exec.Cmd
+}{
+ byPID: make(map[int]*exec.Cmd),
+}
+
+func registerExtractorCommand(cmd *exec.Cmd) {
+ if cmd == nil || cmd.Process == nil {
+ return
+ }
+
+ activeExtractorCommands.Lock()
+ activeExtractorCommands.byPID[cmd.Process.Pid] = cmd
+ activeExtractorCommands.Unlock()
+}
+
+func unregisterExtractorCommand(cmd *exec.Cmd) {
+ if cmd == nil || cmd.Process == nil {
+ return
+ }
+
+ activeExtractorCommands.Lock()
+ delete(activeExtractorCommands.byPID, cmd.Process.Pid)
+ activeExtractorCommands.Unlock()
+}
+
+func runTrackedExtractorCommand(cmd *exec.Cmd) ([]byte, error) {
+ var output bytes.Buffer
+ cmd.Stdout = &output
+ cmd.Stderr = &output
+
+ hideWindow(cmd)
+ if err := cmd.Start(); err != nil {
+ return nil, err
+ }
+
+ registerExtractorCommand(cmd)
+ defer unregisterExtractorCommand(cmd)
+
+ err := cmd.Wait()
+ return output.Bytes(), err
+}
+
func getExecutableName() string {
if runtime.GOOS == "windows" {
return "xtractor.exe"
@@ -26,26 +72,18 @@ func getLegacyExecutableName() string {
}
func KillAllExtractorProcesses() {
- names := []string{getExecutableName(), getLegacyExecutableName()}
- seen := make(map[string]bool)
+ activeExtractorCommands.Lock()
+ commands := make([]*exec.Cmd, 0, len(activeExtractorCommands.byPID))
+ for _, cmd := range activeExtractorCommands.byPID {
+ commands = append(commands, cmd)
+ }
+ activeExtractorCommands.byPID = make(map[int]*exec.Cmd)
+ activeExtractorCommands.Unlock()
- for _, exeName := range names {
- if seen[exeName] {
- continue
+ for _, cmd := range commands {
+ if cmd != nil && cmd.Process != nil {
+ _ = cmd.Process.Kill()
}
- seen[exeName] = true
-
- var cmd *exec.Cmd
- if runtime.GOOS == "windows" {
-
- cmd = exec.Command("taskkill", "/F", "/IM", exeName)
- } else {
-
- cmd = exec.Command("pkill", "-f", exeName)
- }
-
- hideWindow(cmd)
- cmd.CombinedOutput()
}
}
@@ -506,12 +544,7 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) {
"PYTHONIOENCODING=utf-8",
"PYTHONUTF8=1",
)
- hideWindow(cmd)
- output, err := cmd.CombinedOutput()
-
- if cmd.Process != nil {
- cmd.Process.Kill()
- }
+ output, err := runTrackedExtractorCommand(cmd)
if err != nil {
outputStr := string(output)
@@ -533,7 +566,7 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) {
return nil, fmt.Errorf("json_error: Failed to parse JSON response: %v", err)
}
- var timeline []TimelineEntry
+ timeline := make([]TimelineEntry, 0)
accountInfo := AccountInfo{
Name: req.Username,
Nick: req.Username,
@@ -556,7 +589,6 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) {
if isTextOnly {
- timeline = make([]TimelineEntry, 0)
for _, meta := range cliResponse.Metadata {
if !mediaTweetIDs[int64(meta.TweetID)] {
timeline = append(timeline, convertMetadataToTimelineEntry(meta))
@@ -695,12 +727,7 @@ func ExtractDateRange(req DateRangeRequest) (*TwitterResponse, error) {
"PYTHONIOENCODING=utf-8",
"PYTHONUTF8=1",
)
- hideWindow(cmd)
- output, err := cmd.CombinedOutput()
-
- if cmd.Process != nil {
- cmd.Process.Kill()
- }
+ output, err := runTrackedExtractorCommand(cmd)
if err != nil {
outputStr := string(output)
diff --git a/frontend/eslint.config.js b/frontend/eslint.config.js
index 13d2e6b..b798c52 100644
--- a/frontend/eslint.config.js
+++ b/frontend/eslint.config.js
@@ -5,7 +5,7 @@ import reactRefresh from 'eslint-plugin-react-refresh';
import tseslint from 'typescript-eslint';
import { defineConfig, globalIgnores } from 'eslint/config';
export default defineConfig([
- globalIgnores(['dist']),
+ globalIgnores(['dist', 'wailsjs']),
{
files: ['**/*.{ts,tsx}'],
extends: [
@@ -19,4 +19,10 @@ export default defineConfig([
globals: globals.browser,
},
},
+ {
+ files: ['src/components/ui/badge.tsx', 'src/components/ui/button.tsx'],
+ rules: {
+ 'react-refresh/only-export-components': 'off',
+ },
+ },
]);
diff --git a/frontend/index.html b/frontend/index.html
index 185ac22..3dd9aeb 100644
--- a/frontend/index.html
+++ b/frontend/index.html
@@ -6,7 +6,7 @@
-
+
Twitter/X Media Batch Downloader
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index 0ce4e9e..7edf16c 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -1,10 +1,13 @@
import { useState, useEffect, useRef, useCallback } from "react";
import { TooltipProvider } from "@/components/ui/tooltip";
+import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog";
+import { Button } from "@/components/ui/button";
import { getSettings, applyThemeMode, applyFont } from "@/lib/settings";
import { applyTheme } from "@/lib/themes";
+import { compareVersionNumbers } from "@/lib/version";
import { toastWithSound as toast } from "@/lib/toast-with-sound";
import { logger } from "@/lib/logger";
-import { saveFetchState, getFetchState, clearFetchState, getResumableInfo, stateToResponse, mergeTimelines, saveCursor, getCursor, clearCursor, type FetchState, } from "@/lib/fetch-state";
+import { saveFetchState, getFetchState, clearFetchState, getResumableInfo, mergeTimelines, saveCursor, getCursor, clearCursor, type FetchState, } from "@/lib/fetch-state";
import { TitleBar } from "@/components/TitleBar";
import { Sidebar, type PageType } from "@/components/Sidebar";
import { Header } from "@/components/Header";
@@ -13,10 +16,11 @@ import { MediaList } from "@/components/MediaList";
import { DatabaseView } from "@/components/DatabaseView";
import { SettingsPage } from "@/components/SettingsPage";
import { DebugLoggerPage } from "@/components/DebugLoggerPage";
+import { SupportPage } from "@/components/SupportPage";
import { DependencySetupDialog } from "@/components/DependencySetupDialog";
import type { HistoryItem } from "@/components/FetchHistory";
import type { TwitterResponse } from "@/types/api";
-import { ExtractTimeline, ExtractDateRange, SaveAccountToDBWithStatus, CleanupExtractorProcesses, GetAllAccountsFromDB } from "../wailsjs/go/main/App";
+import { ExtractTimeline, ExtractDateRange, SaveAccountToDBWithStatus, CleanupExtractorProcesses, GetAllAccountsFromDB, GetSavedAccountFromDB } from "../wailsjs/go/main/App";
const HISTORY_KEY = "twitter_media_fetch_history";
const MAX_HISTORY = 10;
const CURRENT_VERSION = __APP_VERSION__;
@@ -24,6 +28,28 @@ const BATCH_SIZE = 200;
function formatNumberWithComma(num: number): string {
return num.toLocaleString();
}
+
+function normalizeTimeline(timeline: TwitterResponse["timeline"] | null | undefined): TwitterResponse["timeline"] {
+ return Array.isArray(timeline) ? timeline : [];
+}
+
+function normalizeTwitterResponse(data: TwitterResponse): TwitterResponse {
+ return {
+ ...data,
+ timeline: normalizeTimeline(data.timeline),
+ };
+}
+
+async function loadSavedAccountResponse(username: string, mediaType: string): Promise {
+ try {
+ const savedResponseJSON = await GetSavedAccountFromDB(username, mediaType);
+ return normalizeTwitterResponse(JSON.parse(savedResponseJSON) as TwitterResponse);
+ }
+ catch {
+ return null;
+ }
+}
+
function App() {
const [currentPage, setCurrentPage] = useState("main");
const [username, setUsername] = useState("");
@@ -48,6 +74,10 @@ function App() {
const [isFetchingAll, setIsFetchingAll] = useState(false);
const [searchMode, setSearchMode] = useState("public");
const [searchPrivateType, setSearchPrivateType] = useState("bookmarks");
+ const [hasUnsavedSettings, setHasUnsavedSettings] = useState(false);
+ const [pendingPageChange, setPendingPageChange] = useState(null);
+ const [showUnsavedChangesDialog, setShowUnsavedChangesDialog] = useState(false);
+ const [resetSettingsFn, setResetSettingsFn] = useState<(() => void) | null>(null);
const stopAllRef = useRef(false);
const accountTimersRef = useRef