+# Twitter/X Media Batch Downloader
A GUI tool to download original-quality images and videos from Twitter/X accounts, powered by gallery-dl by @mikf
@@ -10,18 +6,14 @@ A GUI tool to download original-quality images and videos from Twitter/X account


-
-
### [Download](https://github.com/afkarxyz/Twitter-X-Media-Batch-Downloader/releases)
+
+
- If you're familiar with userscripts, install it [**here**](https://greasyfork.org/en/scripts/551198)
- To install the Chrome extension version, click [**here**](https://chromewebstore.google.com/detail/aboccmnimgpjehlplnanplbhlccohhhn)
-## Screenshot
-
-
-
## How to Obtain Auth Token
> [!Warning]
@@ -40,5 +32,3 @@ A GUI tool to download original-quality images and videos from Twitter/X account
6. Do not log out of your account, as a new `auth_token` will be generated by Twitter

-
-[](https://ko-fi.com/afkarxyz)
diff --git a/app.go b/app.go
index 2f2d64b..d4afc7a 100644
--- a/app.go
+++ b/app.go
@@ -10,42 +10,36 @@ import (
"github.com/wailsapp/wails/v2/pkg/runtime"
)
-// App struct
type App struct {
ctx context.Context
downloadCtx context.Context
downloadCancel context.CancelFunc
}
-// NewApp creates a new App application struct
func NewApp() *App {
return &App{}
}
-// startup is called when the app starts. The context is saved
-// so we can call the runtime methods
func (a *App) startup(ctx context.Context) {
a.ctx = ctx
- // Initialize database
- backend.InitDB()
- // Kill any leftover extractor processes from previous session
+
+ if err := backend.InitDB(); err != nil {
+ runtime.LogErrorf(ctx, "failed to initialize database: %v", err)
+ }
+
backend.KillAllExtractorProcesses()
}
-// shutdown is called when the app is closing
func (a *App) shutdown(ctx context.Context) {
backend.CloseDB()
- // Kill any running extractor processes
+
backend.KillAllExtractorProcesses()
}
-// CleanupExtractorProcesses kills all running extractor processes
-// Can be called from frontend when user wants to stop/reset
func (a *App) CleanupExtractorProcesses() {
backend.KillAllExtractorProcesses()
}
-// TimelineRequest represents the request structure for timeline extraction
type TimelineRequest struct {
Username string `json:"username"`
AuthToken string `json:"auth_token"`
@@ -54,10 +48,9 @@ type TimelineRequest struct {
Page int `json:"page"`
MediaType string `json:"media_type"`
Retweets bool `json:"retweets"`
- Cursor string `json:"cursor,omitempty"` // Resume from this cursor position
+ Cursor string `json:"cursor,omitempty"`
}
-// DateRangeRequest represents the request structure for date range extraction
type DateRangeRequest struct {
Username string `json:"username"`
AuthToken string `json:"auth_token"`
@@ -67,9 +60,8 @@ type DateRangeRequest struct {
Retweets bool `json:"retweets"`
}
-// ExtractTimeline extracts media from user timeline
func (a *App) ExtractTimeline(req TimelineRequest) (string, error) {
- // Username not required for bookmarks only
+
if req.Username == "" && req.TimelineType != "bookmarks" {
return "", fmt.Errorf("username is required")
}
@@ -101,7 +93,6 @@ func (a *App) ExtractTimeline(req TimelineRequest) (string, error) {
return string(jsonData), nil
}
-// ExtractDateRange extracts media based on date range
func (a *App) ExtractDateRange(req DateRangeRequest) (string, error) {
if req.Username == "" {
return "", fmt.Errorf("username is required")
@@ -138,13 +129,11 @@ func (a *App) ExtractDateRange(req DateRangeRequest) (string, error) {
return string(jsonData), nil
}
-// OpenFolder opens a folder in the file explorer
func (a *App) OpenFolder(path string) error {
if path == "" {
return fmt.Errorf("path is required")
}
- // Clean the path to use correct separators for the OS
cleanPath := filepath.Clean(path)
err := backend.OpenFolderInExplorer(cleanPath)
@@ -155,51 +144,52 @@ func (a *App) OpenFolder(path string) error {
return nil
}
-// SelectFolder opens a folder selection dialog and returns the selected path
func (a *App) SelectFolder(defaultPath string) (string, error) {
return backend.SelectFolderDialog(a.ctx, defaultPath)
}
-// GetDefaults returns the default configuration
func (a *App) GetDefaults() map[string]string {
return map[string]string{
"downloadPath": backend.GetDefaultDownloadPath(),
}
}
-// Quit closes the application
+func (a *App) IsExtractorInstalled() bool {
+ return backend.IsExtractorInstalled()
+}
+
+func (a *App) DownloadExtractor() error {
+ return backend.DownloadExtractor(nil)
+}
+
func (a *App) Quit() {
panic("quit")
}
-// DownloadMediaRequest represents the request for downloading media (legacy)
type DownloadMediaRequest struct {
URLs []string `json:"urls"`
OutputDir string `json:"output_dir"`
Username string `json:"username"`
- Proxy string `json:"proxy,omitempty"` // Optional proxy URL (e.g., http://proxy:port or socks5://proxy:port)
+ Proxy string `json:"proxy,omitempty"`
}
-// MediaItemRequest represents a media item with metadata
type MediaItemRequest struct {
URL string `json:"url"`
Date string `json:"date"`
TweetID backend.TweetIDString `json:"tweet_id"`
Type string `json:"type"`
- Content string `json:"content,omitempty"` // Tweet text content (for text-only tweets)
- OriginalFilename string `json:"original_filename,omitempty"` // Original filename from API
- AuthorUsername string `json:"author_username,omitempty"` // Username of tweet author (for bookmarks and likes)
+ Content string `json:"content,omitempty"`
+ OriginalFilename string `json:"original_filename,omitempty"`
+ AuthorUsername string `json:"author_username,omitempty"`
}
-// DownloadMediaWithMetadataRequest represents the request for downloading media with metadata
type DownloadMediaWithMetadataRequest struct {
Items []MediaItemRequest `json:"items"`
OutputDir string `json:"output_dir"`
Username string `json:"username"`
- Proxy string `json:"proxy,omitempty"` // Optional proxy URL (e.g., http://proxy:port or socks5://proxy:port)
+ Proxy string `json:"proxy,omitempty"`
}
-// DownloadMediaResponse represents the response for download operation
type DownloadMediaResponse struct {
Success bool `json:"success"`
Downloaded int `json:"downloaded"`
@@ -208,7 +198,6 @@ type DownloadMediaResponse struct {
Message string `json:"message"`
}
-// DownloadMedia downloads media files from URLs (legacy)
func (a *App) DownloadMedia(req DownloadMediaRequest) (DownloadMediaResponse, error) {
if len(req.URLs) == 0 {
return DownloadMediaResponse{
@@ -222,7 +211,6 @@ func (a *App) DownloadMedia(req DownloadMediaRequest) (DownloadMediaResponse, er
outputDir = backend.GetDefaultDownloadPath()
}
- // Create subfolder for username if provided
if req.Username != "" {
outputDir = filepath.Join(outputDir, req.Username)
}
@@ -247,21 +235,18 @@ func (a *App) DownloadMedia(req DownloadMediaRequest) (DownloadMediaResponse, er
}, nil
}
-// DownloadProgress represents download progress event data
type DownloadProgress struct {
Current int `json:"current"`
Total int `json:"total"`
Percent int `json:"percent"`
}
-// DownloadItemStatus represents per-item download status event data
type DownloadItemStatus struct {
TweetID int64 `json:"tweet_id"`
Index int `json:"index"`
- Status string `json:"status"` // "success", "failed", "skipped"
+ Status string `json:"status"`
}
-// DownloadMediaWithMetadata downloads media files with proper naming and categorization
func (a *App) DownloadMediaWithMetadata(req DownloadMediaWithMetadataRequest) (DownloadMediaResponse, error) {
if len(req.Items) == 0 {
return DownloadMediaResponse{
@@ -275,18 +260,15 @@ func (a *App) DownloadMediaWithMetadata(req DownloadMediaWithMetadataRequest) (D
outputDir = backend.GetDefaultDownloadPath()
}
- // Convert request items to backend items
- // For bookmarks and likes, use author_username from each item if available
items := make([]backend.MediaItem, len(req.Items))
for i, item := range req.Items {
- // Use original filename from API if available, otherwise extract from URL
+
originalFilename := item.OriginalFilename
if originalFilename == "" {
- // Fallback: extract from URL if not provided in API response
+
originalFilename = backend.ExtractOriginalFilename(item.URL)
}
- // For bookmarks and likes, use author_username from item, otherwise use req.Username
username := req.Username
if item.AuthorUsername != "" {
username = item.AuthorUsername
@@ -303,10 +285,8 @@ func (a *App) DownloadMediaWithMetadata(req DownloadMediaWithMetadataRequest) (D
}
}
- // Create cancellable context
a.downloadCtx, a.downloadCancel = context.WithCancel(context.Background())
- // Progress callback
progressCallback := func(current, total int) {
percent := 0
if total > 0 {
@@ -319,7 +299,6 @@ func (a *App) DownloadMediaWithMetadata(req DownloadMediaWithMetadataRequest) (D
})
}
- // Per-item status callback
itemStatusCallback := func(tweetID int64, index int, status string) {
runtime.EventsEmit(a.ctx, "download-item-status", DownloadItemStatus{
TweetID: tweetID,
@@ -339,7 +318,6 @@ func (a *App) DownloadMediaWithMetadata(req DownloadMediaWithMetadataRequest) (D
}, err
}
- // Clear cancel function
a.downloadCancel = nil
return DownloadMediaResponse{
@@ -351,7 +329,6 @@ func (a *App) DownloadMediaWithMetadata(req DownloadMediaWithMetadataRequest) (D
}, nil
}
-// StopDownload cancels the current download operation
func (a *App) StopDownload() bool {
if a.downloadCancel != nil {
a.downloadCancel()
@@ -361,24 +338,18 @@ func (a *App) StopDownload() bool {
return false
}
-// Database functions
-
-// SaveAccountToDB saves account data to database
func (a *App) SaveAccountToDB(username, name, profileImage string, totalMedia int, responseJSON string, mediaType string) error {
return backend.SaveAccount(username, name, profileImage, totalMedia, responseJSON, mediaType)
}
-// SaveAccountToDBWithStatus saves account data with cursor and completion status for resume capability
func (a *App) SaveAccountToDBWithStatus(username, name, profileImage string, totalMedia int, responseJSON string, mediaType string, cursor string, completed bool) error {
return backend.SaveAccountWithStatus(username, name, profileImage, totalMedia, responseJSON, mediaType, cursor, completed)
}
-// GetAllAccountsFromDB returns all saved accounts
func (a *App) GetAllAccountsFromDB() ([]backend.AccountListItem, error) {
return backend.GetAllAccounts()
}
-// GetAccountFromDB returns account data by ID
func (a *App) GetAccountFromDB(id int64) (string, error) {
acc, err := backend.GetAccountByID(id)
if err != nil {
@@ -387,67 +358,53 @@ func (a *App) GetAccountFromDB(id int64) (string, error) {
return acc.ResponseJSON, nil
}
-// DeleteAccountFromDB deletes an account from database
func (a *App) DeleteAccountFromDB(id int64) error {
return backend.DeleteAccount(id)
}
-// ClearAllAccountsFromDB deletes all accounts from database
func (a *App) ClearAllAccountsFromDB() error {
return backend.ClearAllAccounts()
}
-// ExportAccountJSON exports account to JSON file in specified directory
func (a *App) ExportAccountJSON(id int64, outputDir string) (string, error) {
return backend.ExportAccountToFile(id, outputDir)
}
-// ExportAccountsTXT exports selected accounts to TXT file in specified directory
func (a *App) ExportAccountsTXT(ids []int64, outputDir string) (string, error) {
return backend.ExportAccountsToTXT(ids, outputDir)
}
-// UpdateAccountGroup updates the group for an account
func (a *App) UpdateAccountGroup(id int64, groupName, groupColor string) error {
return backend.UpdateAccountGroup(id, groupName, groupColor)
}
-// GetAllGroups returns all unique groups
func (a *App) GetAllGroups() ([]map[string]string, error) {
return backend.GetAllGroups()
}
-// FFmpeg functions
-
-// IsFFmpegInstalled checks if ffmpeg is available
func (a *App) IsFFmpegInstalled() bool {
return backend.IsFFmpegInstalled()
}
-// DownloadFFmpeg downloads ffmpeg binary
func (a *App) DownloadFFmpeg() error {
return backend.DownloadFFmpeg(nil)
}
-// IsExifToolInstalled checks if exiftool is available
func (a *App) IsExifToolInstalled() bool {
return backend.IsExifToolInstalled()
}
-// DownloadExifTool downloads exiftool binary
func (a *App) DownloadExifTool() error {
return backend.DownloadExifTool(nil)
}
-// ConvertGIFsRequest represents request for converting GIFs
type ConvertGIFsRequest struct {
FolderPath string `json:"folder_path"`
- Quality string `json:"quality"` // "fast" or "better"
- Resolution string `json:"resolution"` // "original", "high", "medium", "low"
+ Quality string `json:"quality"`
+ Resolution string `json:"resolution"`
DeleteOriginal bool `json:"delete_original"`
}
-// ConvertGIFsResponse represents response for GIF conversion
type ConvertGIFsResponse struct {
Success bool `json:"success"`
Converted int `json:"converted"`
@@ -455,7 +412,6 @@ type ConvertGIFsResponse struct {
Message string `json:"message"`
}
-// ConvertGIFs converts MP4 files in gifs folder to actual GIF format
func (a *App) ConvertGIFs(req ConvertGIFsRequest) (ConvertGIFsResponse, error) {
if !backend.IsFFmpegInstalled() {
return ConvertGIFsResponse{
@@ -464,7 +420,6 @@ func (a *App) ConvertGIFs(req ConvertGIFsRequest) (ConvertGIFsResponse, error) {
}, nil
}
- // Default values if not provided
quality := req.Quality
if quality == "" {
quality = "fast"
@@ -490,41 +445,34 @@ func (a *App) ConvertGIFs(req ConvertGIFsRequest) (ConvertGIFsResponse, error) {
}, nil
}
-// ImportAccountResponse represents the response for import operation
type ImportAccountResponse struct {
Success bool `json:"success"`
Username string `json:"username"`
Message string `json:"message"`
}
-// CheckFolderExists checks if a folder exists for the given username
func (a *App) CheckFolderExists(basePath, username string) bool {
return backend.CheckFolderExists(basePath, username)
}
-// CheckGifsFolderExists checks if a gifs subfolder exists for the given username
func (a *App) CheckGifsFolderExists(basePath, username string) bool {
return backend.CheckGifsFolderExists(basePath, username)
}
-// CheckGifsFolderHasMP4 checks if the gifs folder has any MP4 files to convert
func (a *App) CheckGifsFolderHasMP4(basePath, username string) bool {
return backend.CheckGifsFolderHasMP4(basePath, username)
}
-// GetFolderPath returns the full path for a username folder
func (a *App) GetFolderPath(basePath, username string) string {
return backend.GetFolderPath(basePath, username)
}
-// GetGifsFolderPath returns the full path for a username's gifs folder
func (a *App) GetGifsFolderPath(basePath, username string) string {
return backend.GetGifsFolderPath(basePath, username)
}
-// ImportAccountFromJSON imports account from JSON file (supports both old and new format)
func (a *App) ImportAccountFromJSON() (ImportAccountResponse, error) {
- // Open file dialog
+
filePath, err := runtime.OpenFileDialog(a.ctx, runtime.OpenDialogOptions{
Title: "Import Account JSON",
Filters: []runtime.FileFilter{
@@ -535,12 +483,10 @@ func (a *App) ImportAccountFromJSON() (ImportAccountResponse, error) {
return ImportAccountResponse{Success: false, Message: err.Error()}, err
}
- // User cancelled
if filePath == "" {
return ImportAccountResponse{Success: false, Message: "Cancelled"}, nil
}
- // Import the file
username, err := backend.ImportAccountFromFile(filePath)
if err != nil {
return ImportAccountResponse{Success: false, Message: err.Error()}, err
diff --git a/backend/cmd_unix.go b/backend/cmd_unix.go
index b2c613a..54359d9 100644
--- a/backend/cmd_unix.go
+++ b/backend/cmd_unix.go
@@ -4,7 +4,6 @@ package backend
import "os/exec"
-// hideWindow is a no-op on non-Windows platforms
func hideWindow(cmd *exec.Cmd) {
- // No action needed on Unix-like systems
+
}
diff --git a/backend/cmd_windows.go b/backend/cmd_windows.go
index 1cd228f..122d354 100644
--- a/backend/cmd_windows.go
+++ b/backend/cmd_windows.go
@@ -7,10 +7,9 @@ import (
"syscall"
)
-// hideWindow sets the command to run without showing a console window on Windows
func hideWindow(cmd *exec.Cmd) {
cmd.SysProcAttr = &syscall.SysProcAttr{
HideWindow: true,
- CreationFlags: 0x08000000, // CREATE_NO_WINDOW
+ CreationFlags: 0x08000000,
}
}
diff --git a/backend/config.go b/backend/config.go
index 878aa0c..f6fc8f3 100644
--- a/backend/config.go
+++ b/backend/config.go
@@ -10,21 +10,18 @@ import (
)
func GetDefaultDownloadPath() string {
- // Get user's home directory
+
homeDir, err := os.UserHomeDir()
if err != nil {
- // Fallback to Public Pictures if can't get home dir
+
return "C:\\Users\\Public\\Pictures"
}
- // Return path to user's Pictures folder
return filepath.Join(homeDir, "Pictures")
}
-// GetProxyURL gets proxy URL from environment variables or custom proxy setting
-// Priority: customProxy > HTTP_PROXY/HTTPS_PROXY > http_proxy/https_proxy
func GetProxyURL(customProxy string) (*url.URL, error) {
- // First, check custom proxy setting
+
if customProxy != "" {
proxyURL, err := url.Parse(customProxy)
if err != nil {
@@ -33,7 +30,6 @@ func GetProxyURL(customProxy string) (*url.URL, error) {
return proxyURL, nil
}
- // Check environment variables (case-insensitive on Windows)
proxyEnv := os.Getenv("HTTPS_PROXY")
if proxyEnv == "" {
proxyEnv = os.Getenv("https_proxy")
@@ -53,10 +49,9 @@ func GetProxyURL(customProxy string) (*url.URL, error) {
return proxyURL, nil
}
- return nil, nil // No proxy
+ return nil, nil
}
-// CreateHTTPClient creates an HTTP client with proxy support
func CreateHTTPClient(customProxy string, timeout time.Duration) (*http.Client, error) {
proxyURL, err := GetProxyURL(customProxy)
if err != nil {
diff --git a/backend/database.go b/backend/database.go
index 4e3599e..ec2fcf0 100644
--- a/backend/database.go
+++ b/backend/database.go
@@ -8,10 +8,9 @@ import (
"path/filepath"
"time"
- _ "github.com/mattn/go-sqlite3"
+ _ "modernc.org/sqlite"
)
-// AccountDB represents a saved account in the database
type AccountDB struct {
ID int64 `json:"id"`
Username string `json:"username"`
@@ -25,7 +24,6 @@ type AccountDB struct {
Completed bool `json:"completed"`
}
-// AccountListItem represents a simplified account for listing
type AccountListItem struct {
ID int64 `json:"id"`
Username string `json:"username"`
@@ -44,7 +42,6 @@ type AccountListItem struct {
var db *sql.DB
-// GetDBPath returns the database file path
func GetDBPath() string {
homeDir, err := os.UserHomeDir()
if err != nil {
@@ -53,23 +50,24 @@ func GetDBPath() string {
return filepath.Join(homeDir, ".twitterxmediabatchdownloader", "accounts.db")
}
-// InitDB initializes the database connection
func InitDB() error {
dbPath := GetDBPath()
- // Create directory if not exists
dir := filepath.Dir(dbPath)
if err := os.MkdirAll(dir, 0755); err != nil {
return err
}
var err error
- db, err = sql.Open("sqlite3", dbPath)
+ db, err = sql.Open("sqlite", dbPath)
if err != nil {
return err
}
- // Create tables
+ if err := db.Ping(); err != nil {
+ return err
+ }
+
_, err = db.Exec(`
CREATE TABLE IF NOT EXISTS accounts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -91,33 +89,27 @@ func InitDB() error {
return err
}
- // Add group columns if they don't exist (migration for existing databases)
db.Exec("ALTER TABLE accounts ADD COLUMN group_name TEXT DEFAULT ''")
db.Exec("ALTER TABLE accounts ADD COLUMN group_color TEXT DEFAULT ''")
db.Exec("ALTER TABLE accounts ADD COLUMN media_type TEXT DEFAULT 'all'")
db.Exec("ALTER TABLE accounts ADD COLUMN cursor TEXT DEFAULT ''")
db.Exec("ALTER TABLE accounts ADD COLUMN completed INTEGER DEFAULT 1")
- // Migration: Update unique constraint for existing databases
- // This allows same username with different media types
db.Exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_username_media_type ON accounts(username, media_type)")
return nil
}
-// CloseDB closes the database connection
func CloseDB() {
if db != nil {
db.Close()
}
}
-// SaveAccount saves or updates an account in the database
func SaveAccount(username, name, profileImage string, totalMedia int, responseJSON string, mediaType string) error {
return SaveAccountWithStatus(username, name, profileImage, totalMedia, responseJSON, mediaType, "", true)
}
-// SaveAccountWithStatus saves or updates an account with cursor and completion status
func SaveAccountWithStatus(username, name, profileImage string, totalMedia int, responseJSON string, mediaType string, cursor string, completed bool) error {
if db == nil {
if err := InitDB(); err != nil {
@@ -125,7 +117,6 @@ func SaveAccountWithStatus(username, name, profileImage string, totalMedia int,
}
}
- // Default to "all" if not specified
if mediaType == "" {
mediaType = "all"
}
@@ -151,7 +142,6 @@ func SaveAccountWithStatus(username, name, profileImage string, totalMedia int,
return err
}
-// GetAllAccounts returns all saved accounts
func GetAllAccounts() ([]AccountListItem, error) {
if db == nil {
if err := InitDB(); err != nil {
@@ -185,7 +175,6 @@ func GetAllAccounts() ([]AccountListItem, error) {
acc.LastFetched = lastFetched.Format("2006-01-02 15:04")
acc.Completed = completedInt == 1
- // Extract followers_count and statuses_count from response_json
if responseJSON != "" {
var parsed map[string]interface{}
if err := json.Unmarshal([]byte(responseJSON), &parsed); err == nil {
@@ -206,7 +195,6 @@ func GetAllAccounts() ([]AccountListItem, error) {
return accounts, nil
}
-// UpdateAccountGroup updates the group for an account
func UpdateAccountGroup(id int64, groupName, groupColor string) error {
if db == nil {
if err := InitDB(); err != nil {
@@ -218,7 +206,6 @@ func UpdateAccountGroup(id int64, groupName, groupColor string) error {
return err
}
-// GetAllGroups returns all unique groups
func GetAllGroups() ([]map[string]string, error) {
if db == nil {
if err := InitDB(); err != nil {
@@ -249,7 +236,6 @@ func GetAllGroups() ([]map[string]string, error) {
return groups, nil
}
-// ClearAllAccounts deletes all accounts from the database
func ClearAllAccounts() error {
if db == nil {
if err := InitDB(); err != nil {
@@ -261,7 +247,6 @@ func ClearAllAccounts() error {
return err
}
-// GetAccountByID returns a specific account by ID
func GetAccountByID(id int64) (*AccountDB, error) {
if db == nil {
if err := InitDB(); err != nil {
@@ -284,7 +269,6 @@ func GetAccountByID(id int64) (*AccountDB, error) {
acc.LastFetched = lastFetched
acc.Completed = completedInt == 1
- // Convert legacy format if needed
if converted, err := ConvertLegacyToNewFormat(acc.ResponseJSON); err == nil {
acc.ResponseJSON = converted
}
@@ -292,7 +276,6 @@ func GetAccountByID(id int64) (*AccountDB, error) {
return &acc, nil
}
-// DeleteAccount deletes an account from the database
func DeleteAccount(id int64) error {
if db == nil {
if err := InitDB(); err != nil {
@@ -304,7 +287,6 @@ func DeleteAccount(id int64) error {
return err
}
-// LegacyMediaEntry represents media entry in old format
type LegacyMediaEntry struct {
TweetID string `json:"tweet_id"`
URL string `json:"url"`
@@ -312,7 +294,6 @@ type LegacyMediaEntry struct {
Type string `json:"type"`
}
-// LegacyAccountFormat represents the old saved account format
type LegacyAccountFormat struct {
Username string `json:"username"`
Nick string `json:"nick"`
@@ -327,20 +308,17 @@ type LegacyAccountFormat struct {
MediaList []LegacyMediaEntry `json:"media_list"`
}
-// ConvertLegacyToNewFormat converts old format to new TwitterResponse format
func ConvertLegacyToNewFormat(jsonStr string) (string, error) {
- // First check if it's already in new format (has account_info key)
+
var check map[string]interface{}
if err := json.Unmarshal([]byte(jsonStr), &check); err != nil {
return jsonStr, err
}
- // If already has account_info, it's new format - return as is
if _, hasAccountInfo := check["account_info"]; hasAccountInfo {
return jsonStr, nil
}
- // Check if it's legacy format (has username and media_list)
if _, hasUsername := check["username"]; !hasUsername {
return jsonStr, nil
}
@@ -348,13 +326,11 @@ func ConvertLegacyToNewFormat(jsonStr string) (string, error) {
return jsonStr, nil
}
- // Parse as legacy format
var legacy LegacyAccountFormat
if err := json.Unmarshal([]byte(jsonStr), &legacy); err != nil {
return jsonStr, err
}
- // Convert timeline entries
timeline := make([]map[string]interface{}, len(legacy.MediaList))
for i, media := range legacy.MediaList {
timeline[i] = map[string]interface{}{
@@ -366,7 +342,6 @@ func ConvertLegacyToNewFormat(jsonStr string) (string, error) {
}
}
- // Build new format
newFormat := map[string]interface{}{
"account_info": map[string]interface{}{
"name": legacy.Username,
@@ -387,7 +362,6 @@ func ConvertLegacyToNewFormat(jsonStr string) (string, error) {
},
}
- // Convert back to JSON
newJSON, err := json.Marshal(newFormat)
if err != nil {
return jsonStr, err
@@ -396,20 +370,17 @@ func ConvertLegacyToNewFormat(jsonStr string) (string, error) {
return string(newJSON), nil
}
-// ExportAccountToFile exports account JSON to a file
func ExportAccountToFile(id int64, outputDir string) (string, error) {
acc, err := GetAccountByID(id)
if err != nil {
return "", err
}
- // Create export directory if not exists
exportDir := filepath.Join(outputDir, "twitterxmediabatchdownloader_backups")
if err := os.MkdirAll(exportDir, 0755); err != nil {
return "", err
}
- // Use username (nick) for filename
filename := acc.Username
if filename == "" {
filename = acc.Name
@@ -424,24 +395,21 @@ func ExportAccountToFile(id int64, outputDir string) (string, error) {
return filePath, nil
}
-// ExportAccountsToTXT exports selected accounts to TXT file (one username per line)
func ExportAccountsToTXT(ids []int64, outputDir string) (string, error) {
if len(ids) == 0 {
return "", fmt.Errorf("no accounts to export")
}
- // Create export directory if not exists
exportDir := filepath.Join(outputDir, "twitterxmediabatchdownloader_backups")
if err := os.MkdirAll(exportDir, 0755); err != nil {
return "", err
}
- // Get all accounts by IDs
var usernames []string
for _, id := range ids {
acc, err := GetAccountByID(id)
if err != nil {
- continue // Skip if account not found
+ continue
}
if acc.Username != "" {
usernames = append(usernames, acc.Username)
@@ -452,7 +420,6 @@ func ExportAccountsToTXT(ids []int64, outputDir string) (string, error) {
return "", fmt.Errorf("no valid usernames found")
}
- // Create TXT content (one username per line)
txtContent := ""
for i, username := range usernames {
if i > 0 {
@@ -461,7 +428,6 @@ func ExportAccountsToTXT(ids []int64, outputDir string) (string, error) {
txtContent += username
}
- // Filename: twitterxmediabatchdownloader_multiple.txt
filePath := filepath.Join(exportDir, "twitterxmediabatchdownloader_multiple.txt")
if err := os.WriteFile(filePath, []byte(txtContent), 0644); err != nil {
@@ -471,7 +437,6 @@ func ExportAccountsToTXT(ids []int64, outputDir string) (string, error) {
return filePath, nil
}
-// ImportAccountFromFile imports account from JSON file (supports both old and new format)
func ImportAccountFromFile(filePath string) (string, error) {
if db == nil {
if err := InitDB(); err != nil {
@@ -479,7 +444,6 @@ func ImportAccountFromFile(filePath string) (string, error) {
}
}
- // Read file
data, err := os.ReadFile(filePath)
if err != nil {
return "", err
@@ -487,19 +451,16 @@ func ImportAccountFromFile(filePath string) (string, error) {
jsonStr := string(data)
- // Convert legacy format if needed
convertedJSON, err := ConvertLegacyToNewFormat(jsonStr)
if err != nil {
return "", err
}
- // Parse the converted JSON to extract account info
var parsed map[string]interface{}
if err := json.Unmarshal([]byte(convertedJSON), &parsed); err != nil {
return "", err
}
- // Extract account info
accountInfo, ok := parsed["account_info"].(map[string]interface{})
if !ok {
return "", fmt.Errorf("invalid JSON format: missing account_info")
@@ -518,7 +479,6 @@ func ImportAccountFromFile(filePath string) (string, error) {
return "", fmt.Errorf("invalid JSON format: missing username")
}
- // Save to database with default media type "all" for imported files
err = SaveAccount(username, name, profileImage, totalURLs, convertedJSON, "all")
if err != nil {
return "", err
diff --git a/backend/download.go b/backend/download.go
index 9565d84..f1a74d6 100644
--- a/backend/download.go
+++ b/backend/download.go
@@ -15,32 +15,28 @@ import (
)
const (
- // MaxConcurrentDownloads is the number of parallel downloads
MaxConcurrentDownloads = 10
)
-// MediaItem represents a media item with metadata for download
type MediaItem struct {
- URL string `json:"url"`
- Date string `json:"date"`
- TweetID int64 `json:"tweet_id"`
- Type string `json:"type"`
- Username string `json:"username"`
- Content string `json:"content,omitempty"` // Tweet text content (for text-only tweets)
- OriginalFilename string `json:"original_filename,omitempty"` // Original Twitter media filename (15 char alphanumeric)
+ URL string `json:"url"`
+ Date string `json:"date"`
+ TweetID int64 `json:"tweet_id"`
+ Type string `json:"type"`
+ Username string `json:"username"`
+ Content string `json:"content,omitempty"`
+ OriginalFilename string `json:"original_filename,omitempty"`
}
-// DownloadMediaFiles downloads media files from URLs to the output directory (legacy)
func DownloadMediaFiles(urls []string, outputDir string, customProxy string) (downloaded int, failed int, err error) {
- // Create output directory if it doesn't exist
+
if err := os.MkdirAll(outputDir, 0755); err != nil {
return 0, len(urls), fmt.Errorf("failed to create output directory: %v", err)
}
- // Create HTTP client with proxy support
client, err := CreateHTTPClient(customProxy, 60*time.Second)
if err != nil {
- // If proxy setup fails, use default client without proxy
+
client = &http.Client{
Timeout: 60 * time.Second,
}
@@ -50,7 +46,6 @@ func DownloadMediaFiles(urls []string, outputDir string, customProxy string) (do
filename := extractFilename(mediaURL)
outputPath := filepath.Join(outputDir, filename)
- // Skip if file already exists
if _, err := os.Stat(outputPath); err == nil {
downloaded++
continue
@@ -66,21 +61,16 @@ func DownloadMediaFiles(urls []string, outputDir string, customProxy string) (do
return downloaded, failed, nil
}
-// ProgressCallback is a function type for progress updates
type ProgressCallback func(current, total int)
-// ItemStatusCallback is a function type for per-item status updates
type ItemStatusCallback func(tweetID int64, index int, status string) // status: "success", "failed", "skipped"
-// downloadTask represents a single download task
type downloadTask struct {
item MediaItem
outputPath string
index int
}
-// DownloadMediaWithMetadataProgressAndStatus downloads media files with progress and per-item status callbacks
-// Returns: downloaded count, skipped count, failed count, error
func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir string, username string, progress ProgressCallback, itemStatus ItemStatusCallback, ctx context.Context, customProxy string) (downloaded int, skipped int, failed int, err error) {
if ctx == nil {
ctx = context.Background()
@@ -91,24 +81,20 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str
return 0, 0, 0, nil
}
- // Prepare all tasks first (sequential to handle tweet media count)
- // For bookmarks and likes, each item may have different username, so we track per username
- tweetMediaCount := make(map[string]map[int64]int) // username -> tweet_id -> count
+ tweetMediaCount := make(map[string]map[int64]int)
tasks := make([]downloadTask, 0, total)
for i, item := range items {
- // Use item.Username if available (for bookmarks/likes with different authors), otherwise use provided username
+
itemUsername := item.Username
if itemUsername == "" {
itemUsername = username
}
- // Initialize tweet media count for this username if needed
if tweetMediaCount[itemUsername] == nil {
tweetMediaCount[itemUsername] = make(map[int64]int)
}
- // Determine subfolder based on type
var subfolder string
switch item.Type {
case "photo":
@@ -123,29 +109,23 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str
subfolder = "other"
}
- // Create base directory for this username
baseDir := filepath.Join(outputDir, itemUsername)
if err := os.MkdirAll(baseDir, 0755); err != nil {
continue
}
- // Create type subfolder
typeDir := filepath.Join(baseDir, subfolder)
if err := os.MkdirAll(typeDir, 0755); err != nil {
continue
}
- // Format timestamp from date
timestamp := formatTimestamp(item.Date)
- // Get file extension
ext := getExtension(item.URL, item.Type)
- // Increment counter for this username and tweet_id
tweetMediaCount[itemUsername][item.TweetID]++
mediaIndex := tweetMediaCount[itemUsername][item.TweetID]
- // Create filename: {username}_{timestamp}_{tweet_id}_{index}.{ext}
filename := fmt.Sprintf("%s_%s_%d_%02d%s", itemUsername, timestamp, item.TweetID, mediaIndex, ext)
outputPath := filepath.Join(typeDir, filename)
@@ -156,27 +136,23 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str
})
}
- // Counters for parallel downloads
var downloadedCount int64
var skippedCount int64
var failedCount int64
var completedCount int64
- // Create worker pool
taskChan := make(chan downloadTask, len(tasks))
var wg sync.WaitGroup
- // Start workers
numWorkers := MaxConcurrentDownloads
if numWorkers > len(tasks) {
numWorkers = len(tasks)
}
- // Create HTTP client once for all workers (shared client is more efficient)
var sharedClient *http.Client
client, err := CreateHTTPClient(customProxy, 60*time.Second)
if err != nil {
- // If proxy setup fails, use default client without proxy
+
sharedClient = &http.Client{
Timeout: 60 * time.Second,
}
@@ -191,7 +167,7 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str
client := sharedClient
for task := range taskChan {
- // Check for cancellation
+
select {
case <-ctx.Done():
return
@@ -199,17 +175,17 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str
}
var status string
- // Skip if file already exists
+
if _, err := os.Stat(task.outputPath); err == nil {
status = "skipped"
- // Emit status immediately for skipped files
+
if itemStatus != nil {
itemStatus(task.item.TweetID, task.index, status)
}
atomic.AddInt64(&skippedCount, 1)
- continue // Skip to next task
+ continue
} else if task.item.Type == "text" {
- // For text tweets, write content to file
+
if err := os.WriteFile(task.outputPath, []byte(task.item.Content), 0644); err != nil {
atomic.AddInt64(&failedCount, 1)
status = "failed"
@@ -221,30 +197,23 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str
atomic.AddInt64(&failedCount, 1)
status = "failed"
} else {
- // Embed metadata after successful download
+
tweetURL := fmt.Sprintf("https://x.com/i/status/%d", task.item.TweetID)
- // Always extract original filename from URL (simpler approach)
+
originalFilename := ExtractOriginalFilename(task.item.URL)
-
- // For debugging: if original filename is still empty for video, it means it's not in the URL
- // This is acceptable - video URLs from Twitter may not contain original filename
-
- // Embed metadata (non-fatal: if it fails, file is still downloaded)
+
if err := EmbedMetadata(task.outputPath, task.item.Content, tweetURL, originalFilename); err != nil {
- // Log error but don't fail the download
- // Metadata embedding is optional
+
}
-
+
atomic.AddInt64(&downloadedCount, 1)
status = "success"
}
- // Emit per-item status
if itemStatus != nil {
itemStatus(task.item.TweetID, task.index, status)
}
- // Update progress
completed := atomic.AddInt64(&completedCount, 1)
if progress != nil {
progress(int(completed), total)
@@ -253,7 +222,6 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str
}()
}
- // Send tasks to workers
for _, task := range tasks {
select {
case <-ctx.Done():
@@ -265,13 +233,11 @@ func DownloadMediaWithMetadataProgressAndStatus(items []MediaItem, outputDir str
}
close(taskChan)
- // Wait for all workers to finish
wg.Wait()
return int(downloadedCount), int(skippedCount), int(failedCount), nil
}
-// downloadFileWithContext downloads a single file with context support for cancellation
func downloadFileWithContext(ctx context.Context, client *http.Client, url, outputPath string) error {
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
@@ -298,14 +264,13 @@ func downloadFileWithContext(ctx context.Context, client *http.Client, url, outp
return err
}
-// formatTimestamp converts date string to timestamp format
func formatTimestamp(dateStr string) string {
- // Try parsing various date formats
+
formats := []string{
- "2006-01-02T15:04:05", // ISO 8601 without timezone (from extractor)
- "2006-01-02T15:04:05+00:00", // ISO 8601 with timezone
- "2006-01-02T15:04:05-07:00", // ISO 8601 with timezone offset
- time.RFC3339, // Standard RFC3339
+ "2006-01-02T15:04:05",
+ "2006-01-02T15:04:05+00:00",
+ "2006-01-02T15:04:05-07:00",
+ time.RFC3339,
"2006-01-02T15:04:05.000Z",
"2006-01-02T15:04:05Z",
"2006-01-02 15:04:05",
@@ -318,35 +283,30 @@ func formatTimestamp(dateStr string) string {
}
}
- // Fallback: use empty string to indicate parsing failed
return "00000000_000000"
}
-// getExtension determines file extension from URL and type
func getExtension(mediaURL string, mediaType string) string {
parsedURL, err := url.Parse(mediaURL)
if err != nil {
return ".jpg"
}
- // Check format query param for Twitter images
if format := parsedURL.Query().Get("format"); format != "" {
return "." + format
}
- // Get extension from path
path := parsedURL.Path
ext := filepath.Ext(path)
if ext != "" {
return ext
}
- // Default based on type
switch mediaType {
case "video":
return ".mp4"
case "gif", "animated_gif":
- return ".mp4" // Twitter GIFs are actually MP4
+ return ".mp4"
case "text":
return ".txt"
default:
@@ -354,28 +314,23 @@ func getExtension(mediaURL string, mediaType string) string {
}
}
-// extractFilename extracts filename from URL (legacy)
func extractFilename(mediaURL string) string {
parsedURL, err := url.Parse(mediaURL)
if err != nil {
return fmt.Sprintf("media_%d", time.Now().UnixNano())
}
- // Get the path part
path := parsedURL.Path
- // Extract base filename
base := filepath.Base(path)
- // Handle Twitter image URLs: /media/XXX -> XXX.jpg
if strings.Contains(mediaURL, "pbs.twimg.com/media/") {
- // Get format from query params
+
format := parsedURL.Query().Get("format")
if format == "" {
format = "jpg"
}
- // Remove any existing extension
if idx := strings.LastIndex(base, "."); idx > 0 {
base = base[:idx]
}
@@ -383,12 +338,10 @@ func extractFilename(mediaURL string) string {
return base + "." + format
}
- // Handle Twitter video URLs
if strings.Contains(mediaURL, "video.twimg.com") {
return base
}
- // Default: use base name or generate one
if base == "" || base == "." {
return fmt.Sprintf("media_%d", time.Now().UnixNano())
}
@@ -396,7 +349,6 @@ func extractFilename(mediaURL string) string {
return base
}
-// downloadFile downloads a single file from URL
func downloadFile(client *http.Client, url, outputPath string) error {
resp, err := client.Get(url)
if err != nil {
diff --git a/backend/embed_unix.go b/backend/embed_unix.go
deleted file mode 100644
index 64c34d9..0000000
--- a/backend/embed_unix.go
+++ /dev/null
@@ -1,8 +0,0 @@
-//go:build linux || darwin
-
-package backend
-
-import _ "embed"
-
-//go:embed bin/extractor
-var extractorBin []byte
diff --git a/backend/embed_windows.go b/backend/embed_windows.go
deleted file mode 100644
index e6121a5..0000000
--- a/backend/embed_windows.go
+++ /dev/null
@@ -1,8 +0,0 @@
-//go:build windows
-
-package backend
-
-import _ "embed"
-
-//go:embed bin/extractor.exe
-var extractorBin []byte
diff --git a/backend/exiftool.go b/backend/exiftool.go
index 639d6a4..823be75 100644
--- a/backend/exiftool.go
+++ b/backend/exiftool.go
@@ -14,26 +14,20 @@ import (
"strings"
)
-// ExifTool download URLs (SourceForge)
const (
- // Windows 64-bit
exiftoolWindows64URL = "https://sourceforge.net/projects/exiftool/files/exiftool-13.43_64.zip/download"
- // Windows 32-bit
+
exiftoolWindows32URL = "https://sourceforge.net/projects/exiftool/files/exiftool-13.43_32.zip/download"
- // Unix (Linux/macOS): tar.gz
+
exiftoolUnixURL = "https://sourceforge.net/projects/exiftool/files/Image-ExifTool-13.43.tar.gz/download"
)
-// ExifTool SHA256 hashes for verification
-// Note: Hashes will be calculated during download if not provided
-// For now, we'll skip hash verification for version 13.43 (can be added later if needed)
const (
- exiftoolWindows64Hash = "" // Hash not provided, will skip verification
- exiftoolWindows32Hash = "" // Hash not provided, will skip verification
- exiftoolUnixHash = "" // Hash not provided, will skip verification
+ exiftoolWindows64Hash = ""
+ exiftoolWindows32Hash = ""
+ exiftoolUnixHash = ""
)
-// GetExifToolPath returns the path to exiftool binary
func GetExifToolPath() string {
homeDir, _ := os.UserHomeDir()
baseDir := filepath.Join(homeDir, ".twitterxmediabatchdownloader")
@@ -42,27 +36,25 @@ func GetExifToolPath() string {
case "windows":
return filepath.Join(baseDir, "exiftool.exe")
default:
- // For Unix (Linux/macOS), exiftool is in Image-ExifTool-VERSION/exiftool
- // We need to find the folder dynamically since version may vary
+
pattern := filepath.Join(baseDir, "Image-ExifTool-*")
matches, err := filepath.Glob(pattern)
if err == nil && len(matches) > 0 {
- // Use the first match (should only be one)
+
exiftoolPath := filepath.Join(matches[0], "exiftool")
if _, err := os.Stat(exiftoolPath); err == nil {
return exiftoolPath
}
}
- // Fallback: return path that will be created after extraction
+
return filepath.Join(baseDir, "exiftool")
}
}
-// IsExifToolInstalled checks if exiftool is available
func IsExifToolInstalled() bool {
exiftoolPath := GetExifToolPath()
if _, err := os.Stat(exiftoolPath); err == nil {
- // Test if it's executable by running version command
+
cmd := exec.Command(exiftoolPath, "-ver")
hideWindow(cmd)
if err := cmd.Run(); err == nil {
@@ -72,7 +64,6 @@ func IsExifToolInstalled() bool {
return false
}
-// calculateSHA256 calculates SHA256 hash of a file
func calculateSHA256(filePath string) (string, error) {
file, err := os.Open(filePath)
if err != nil {
@@ -88,7 +79,6 @@ func calculateSHA256(filePath string) (string, error) {
return hex.EncodeToString(hash.Sum(nil)), nil
}
-// verifyHash verifies the SHA256 hash of a downloaded file
func verifyHash(filePath, expectedHash string) error {
actualHash, err := calculateSHA256(filePath)
if err != nil {
@@ -102,12 +92,10 @@ func verifyHash(filePath, expectedHash string) error {
return nil
}
-// is64Bit checks if the system is 64-bit
func is64Bit() bool {
return runtime.GOARCH == "amd64" || runtime.GOARCH == "arm64"
}
-// DownloadExifTool downloads exiftool binary for current platform
func DownloadExifTool(progressCallback func(downloaded, total int64)) error {
var downloadURL string
var expectedHash string
@@ -122,14 +110,13 @@ func DownloadExifTool(progressCallback func(downloaded, total int64)) error {
expectedHash = exiftoolWindows32Hash
}
case "linux", "darwin":
- // Linux and macOS use the same tar.gz archive
+
downloadURL = exiftoolUnixURL
expectedHash = exiftoolUnixHash
default:
return fmt.Errorf("unsupported platform: %s", runtime.GOOS)
}
- // Create temp file for download
tempFile, err := os.CreateTemp("", "exiftool-*")
if err != nil {
return fmt.Errorf("failed to create temp file: %v", err)
@@ -138,7 +125,6 @@ func DownloadExifTool(progressCallback func(downloaded, total int64)) error {
defer os.Remove(tempPath)
defer tempFile.Close()
- // Download file
resp, err := http.Get(downloadURL)
if err != nil {
return fmt.Errorf("failed to download exiftool: %v", err)
@@ -149,7 +135,6 @@ func DownloadExifTool(progressCallback func(downloaded, total int64)) error {
return fmt.Errorf("failed to download exiftool: status %d", resp.StatusCode)
}
- // Copy with progress
total := resp.ContentLength
var downloaded int64
buf := make([]byte, 32*1024)
@@ -175,14 +160,12 @@ func DownloadExifTool(progressCallback func(downloaded, total int64)) error {
}
tempFile.Close()
- // Verify hash before extraction
if expectedHash != "" {
if err := verifyHash(tempPath, expectedHash); err != nil {
return fmt.Errorf("hash verification failed: %v", err)
}
}
- // Extract exiftool binary
exiftoolPath := GetExifToolPath()
baseDir := filepath.Dir(exiftoolPath)
if err := os.MkdirAll(baseDir, 0755); err != nil {
@@ -193,15 +176,13 @@ func DownloadExifTool(progressCallback func(downloaded, total int64)) error {
case "windows":
return extractExifToolFromZip(tempPath, exiftoolPath)
case "linux", "darwin":
- // For Linux/macOS, we need to extract and build
- // For simplicity, we'll extract the exiftool script from tar.gz
+
return extractExifToolFromTarGz(tempPath, exiftoolPath)
}
return nil
}
-// extractExifToolFromZip extracts exiftool from Windows zip archive
func extractExifToolFromZip(zipPath, destPath string) error {
r, err := zip.OpenReader(zipPath)
if err != nil {
@@ -209,11 +190,10 @@ func extractExifToolFromZip(zipPath, destPath string) error {
}
defer r.Close()
- // Look for exiftool(-k).exe or exiftool.exe in the zip
var exiftoolFile *zip.File
for _, f := range r.File {
name := filepath.Base(f.Name)
- // ExifTool Windows zip contains exiftool(-k).exe
+
if name == "exiftool(-k).exe" || name == "exiftool.exe" {
exiftoolFile = f
break
@@ -240,12 +220,11 @@ func extractExifToolFromZip(zipPath, destPath string) error {
return fmt.Errorf("failed to extract file: %v", err)
}
- // Also need to extract the lib directory for Windows (exiftool_files folder)
libDir := filepath.Join(filepath.Dir(destPath), "exiftool_files")
for _, f := range r.File {
- // Check if file is in exiftool_files directory
+
if strings.Contains(f.Name, "exiftool_files") && !strings.HasSuffix(f.Name, "/") {
- // Extract relative path from exiftool_files
+
parts := strings.Split(f.Name, "exiftool_files/")
if len(parts) < 2 {
continue
@@ -257,21 +236,18 @@ func extractExifToolFromZip(zipPath, destPath string) error {
targetPath := filepath.Join(libDir, relPath)
- // Create directory structure
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
continue
}
- // Skip if it's a directory entry
if f.FileInfo().IsDir() {
os.MkdirAll(targetPath, 0755)
continue
}
- // Extract file
rc, err := f.Open()
if err != nil {
- continue // Skip on error
+ continue
}
outFile, err := os.Create(targetPath)
@@ -289,15 +265,10 @@ func extractExifToolFromZip(zipPath, destPath string) error {
return nil
}
-// extractExifToolFromTarGz extracts exiftool from tar.gz archive (Linux/macOS)
func extractExifToolFromTarGz(tarGzPath, destPath string) error {
- // For Linux/macOS, ExifTool is a Perl script that requires the lib directory
- // We'll extract the entire folder and use exiftool directly from Image-ExifTool-VERSION/exiftool
- // This is simpler than implementing tar.gz extraction in Go
baseDir := filepath.Dir(destPath)
- // Use system tar command to extract
cmd := exec.Command("tar", "-xzf", tarGzPath, "-C", baseDir)
hideWindow(cmd)
output, err := cmd.CombinedOutput()
@@ -305,28 +276,23 @@ func extractExifToolFromTarGz(tarGzPath, destPath string) error {
return fmt.Errorf("failed to extract tar.gz: %v, output: %s", err, string(output))
}
- // Find the extracted exiftool script
- // The archive extracts to Image-ExifTool-VERSION/exiftool
- // We need to find the folder dynamically since version may vary
var extractedDir string
var exiftoolScript string
- // Use glob pattern to find Image-ExifTool-* directories
pattern := filepath.Join(baseDir, "Image-ExifTool-*")
matches, err := filepath.Glob(pattern)
if err == nil && len(matches) > 0 {
- // Use the first match (should only be one)
+
extractedDir = matches[0]
exiftoolScript = filepath.Join(extractedDir, "exiftool")
} else {
- // Fallback: try common version or alternative location
+
extractedDir = filepath.Join(baseDir, "Image-ExifTool-13.43")
exiftoolScript = filepath.Join(extractedDir, "exiftool")
}
- // Check if exiftool script exists
if _, err := os.Stat(exiftoolScript); err != nil {
- // Try alternative location
+
extractedDir = filepath.Join(baseDir, "exiftool")
exiftoolScript = filepath.Join(extractedDir, "exiftool")
if _, err := os.Stat(exiftoolScript); err != nil {
@@ -334,15 +300,9 @@ func extractExifToolFromTarGz(tarGzPath, destPath string) error {
}
}
- // Make exiftool executable (required for Unix systems)
if err := os.Chmod(exiftoolScript, 0755); err != nil {
return fmt.Errorf("failed to make exiftool executable: %v", err)
}
- // For Unix, we keep the entire folder structure and use exiftool directly from it
- // The folder will remain in baseDir and GetExifToolPath() will find it via glob pattern
- // No need to copy or remove anything
-
return nil
}
-
diff --git a/backend/extractor.go b/backend/extractor.go
new file mode 100644
index 0000000..3950844
--- /dev/null
+++ b/backend/extractor.go
@@ -0,0 +1,369 @@
+package backend
+
+import (
+ "archive/zip"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "time"
+)
+
+const extractorReleaseAPIURL = "https://api.github.com/repos/afkarxyz/xtractor-binaries/releases/latest"
+
+type githubRelease struct {
+ TagName string `json:"tag_name"`
+ Assets []githubReleaseAsset `json:"assets"`
+}
+
+type githubReleaseAsset struct {
+ Name string `json:"name"`
+ BrowserDownloadURL string `json:"browser_download_url"`
+ Digest string `json:"digest"`
+}
+
+func IsExtractorInstalled() bool {
+ info, err := os.Stat(getExtractorPath())
+ return err == nil && !info.IsDir()
+}
+
+func DownloadExtractor(progressCallback func(downloaded, total int64)) error {
+ assetName, err := getExtractorAssetName()
+ if err != nil {
+ return err
+ }
+
+ release, err := fetchLatestExtractorRelease()
+ if err != nil {
+ return err
+ }
+
+ asset, err := findExtractorAsset(release.Assets, assetName)
+ if err != nil {
+ return err
+ }
+
+ tempFile, err := os.CreateTemp("", "xtractor-*.zip")
+ if err != nil {
+ return fmt.Errorf("failed to create temp file: %v", err)
+ }
+ tempPath := tempFile.Name()
+ defer os.Remove(tempPath)
+ defer tempFile.Close()
+
+ if err := downloadExtractorArchive(asset.BrowserDownloadURL, tempFile, progressCallback); err != nil {
+ return err
+ }
+
+ if err := tempFile.Close(); err != nil {
+ return fmt.Errorf("failed to finalize xtractor archive: %v", err)
+ }
+
+ if err := verifyDownloadedDigest(tempPath, asset.Digest); err != nil {
+ return err
+ }
+
+ KillAllExtractorProcesses()
+
+ exePath := getExtractorPath()
+ if err := os.MkdirAll(filepath.Dir(exePath), 0755); err != nil {
+ return fmt.Errorf("failed to create xtractor directory: %v", err)
+ }
+
+ if err := extractExtractorArchive(tempPath, exePath); err != nil {
+ return err
+ }
+
+ _ = removeLegacyExtractorBinary()
+
+ return nil
+}
+
+func requireExtractorPath() (string, error) {
+ exePath := getExtractorPath()
+ info, err := os.Stat(exePath)
+ if err != nil || info.IsDir() {
+ return "", fmt.Errorf("xtractor dependency is not installed. Please download it from the startup prompt or Settings > Core Xtractor")
+ }
+
+ if runtime.GOOS != "windows" {
+ _ = os.Chmod(exePath, 0755)
+ }
+
+ return exePath, nil
+}
+
+func getExtractorAssetName() (string, error) {
+ switch runtime.GOOS {
+ case "windows":
+ switch runtime.GOARCH {
+ case "amd64", "arm64":
+ return "windows-amd64.zip", nil
+ }
+ case "linux":
+ if runtime.GOARCH == "amd64" {
+ return "linux-amd64.zip", nil
+ }
+ case "darwin":
+ switch resolveMacOSExtractorArch(runtime.GOARCH, detectMacOSHardwareArch()) {
+ case "amd64":
+ return "macos-amd64.zip", nil
+ case "arm64":
+ return "macos-arm64.zip", nil
+ }
+ }
+
+ return "", fmt.Errorf("unsupported platform for prebuilt xtractor: %s/%s", runtime.GOOS, runtime.GOARCH)
+}
+
+func resolveMacOSExtractorArch(processArch, hardwareArch string) string {
+ if hardwareArch == "arm64" {
+ return "arm64"
+ }
+
+ return processArch
+}
+
+func detectMacOSHardwareArch() string {
+ if runtime.GOOS != "darwin" {
+ return ""
+ }
+
+ // On Apple Silicon, this stays "1" even if the app itself runs under Rosetta.
+ cmd := exec.Command("sysctl", "-in", "hw.optional.arm64")
+ hideWindow(cmd)
+ output, err := cmd.Output()
+ if err != nil {
+ return ""
+ }
+
+ if strings.TrimSpace(string(output)) == "1" {
+ return "arm64"
+ }
+
+ return ""
+}
+
+func fetchLatestExtractorRelease() (*githubRelease, error) {
+ client, err := CreateHTTPClient("", 30*time.Second)
+ if err != nil {
+ return nil, fmt.Errorf("failed to configure xtractor network client: %v", err)
+ }
+
+ req, err := http.NewRequest(http.MethodGet, extractorReleaseAPIURL, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create xtractor release request: %v", err)
+ }
+ req.Header.Set("Accept", "application/vnd.github+json")
+ req.Header.Set("User-Agent", "Twitter-X-Media-Batch-Downloader")
+
+ resp, err := client.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to fetch xtractor release info: %v", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("failed to fetch xtractor release info: status %d", resp.StatusCode)
+ }
+
+ var release githubRelease
+ if err := json.NewDecoder(resp.Body).Decode(&release); err != nil {
+ return nil, fmt.Errorf("failed to decode xtractor release info: %v", err)
+ }
+
+ return &release, nil
+}
+
+func findExtractorAsset(assets []githubReleaseAsset, assetName string) (*githubReleaseAsset, error) {
+ for i := range assets {
+ if assets[i].Name == assetName {
+ return &assets[i], nil
+ }
+ }
+
+ return nil, fmt.Errorf("xtractor asset %q not found in latest release", assetName)
+}
+
+func downloadExtractorArchive(downloadURL string, destination *os.File, progressCallback func(downloaded, total int64)) error {
+ client, err := CreateHTTPClient("", 10*time.Minute)
+ if err != nil {
+ return fmt.Errorf("failed to configure xtractor download client: %v", err)
+ }
+
+ req, err := http.NewRequest(http.MethodGet, downloadURL, nil)
+ if err != nil {
+ return fmt.Errorf("failed to create xtractor download request: %v", err)
+ }
+ req.Header.Set("User-Agent", "Twitter-X-Media-Batch-Downloader")
+
+ resp, err := client.Do(req)
+ if err != nil {
+ return fmt.Errorf("failed to download xtractor: %v", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ return fmt.Errorf("failed to download xtractor: status %d", resp.StatusCode)
+ }
+
+ total := resp.ContentLength
+ var downloaded int64
+ buf := make([]byte, 32*1024)
+
+ for {
+ n, err := resp.Body.Read(buf)
+ if n > 0 {
+ if _, writeErr := destination.Write(buf[:n]); writeErr != nil {
+ return fmt.Errorf("failed to write xtractor archive: %v", writeErr)
+ }
+ downloaded += int64(n)
+ if progressCallback != nil {
+ progressCallback(downloaded, total)
+ }
+ }
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return fmt.Errorf("failed to download xtractor: %v", err)
+ }
+ }
+
+ return nil
+}
+
+func verifyDownloadedDigest(filePath, digest string) error {
+ digest = strings.TrimSpace(digest)
+ if digest == "" {
+ return nil
+ }
+
+ expectedHash := strings.TrimPrefix(strings.ToLower(digest), "sha256:")
+
+ file, err := os.Open(filePath)
+ if err != nil {
+ return fmt.Errorf("failed to open downloaded xtractor archive: %v", err)
+ }
+ defer file.Close()
+
+ hash := sha256.New()
+ if _, err := io.Copy(hash, file); err != nil {
+ return fmt.Errorf("failed to verify xtractor archive: %v", err)
+ }
+
+ actualHash := hex.EncodeToString(hash.Sum(nil))
+ if actualHash != expectedHash {
+ return fmt.Errorf("xtractor archive checksum mismatch")
+ }
+
+ return nil
+}
+
+func extractExtractorArchive(zipPath, destPath string) error {
+ reader, err := zip.OpenReader(zipPath)
+ if err != nil {
+ return fmt.Errorf("failed to open xtractor archive: %v", err)
+ }
+ defer reader.Close()
+
+ entry, err := findExtractorBinaryEntry(reader.File)
+ if err != nil {
+ return err
+ }
+
+ in, err := entry.Open()
+ if err != nil {
+ return fmt.Errorf("failed to open xtractor binary from archive: %v", err)
+ }
+ defer in.Close()
+
+ tempDest := destPath + ".download"
+ removeTemp := true
+ defer func() {
+ if removeTemp {
+ _ = os.Remove(tempDest)
+ }
+ }()
+
+ out, err := os.OpenFile(tempDest, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755)
+ if err != nil {
+ return fmt.Errorf("failed to create xtractor binary: %v", err)
+ }
+
+ if _, err := io.Copy(out, in); err != nil {
+ out.Close()
+ return fmt.Errorf("failed to extract xtractor binary: %v", err)
+ }
+
+ if err := out.Close(); err != nil {
+ return fmt.Errorf("failed to finalize xtractor binary: %v", err)
+ }
+
+ if err := os.Chmod(tempDest, 0755); err != nil && runtime.GOOS != "windows" {
+ return fmt.Errorf("failed to make xtractor executable: %v", err)
+ }
+
+ if err := os.Remove(destPath); err != nil && !os.IsNotExist(err) {
+ return fmt.Errorf("failed to replace existing xtractor: %v", err)
+ }
+
+ if err := os.Rename(tempDest, destPath); err != nil {
+ return fmt.Errorf("failed to install xtractor: %v", err)
+ }
+
+ removeTemp = false
+ return nil
+}
+
+func findExtractorBinaryEntry(entries []*zip.File) (*zip.File, error) {
+ var fallback *zip.File
+
+ for _, entry := range entries {
+ if entry.FileInfo().IsDir() {
+ continue
+ }
+
+ if fallback == nil {
+ fallback = entry
+ }
+
+ name := strings.ToLower(filepath.Base(entry.Name))
+ if strings.Contains(name, "xtractor") || strings.Contains(name, "extractor") {
+ return entry, nil
+ }
+ }
+
+ if fallback != nil {
+ return fallback, nil
+ }
+
+ return nil, fmt.Errorf("xtractor binary not found in archive")
+}
+
+func getLegacyExtractorPath() string {
+ homeDir, _ := os.UserHomeDir()
+ baseDir := filepath.Join(homeDir, ".twitterxmediabatchdownloader")
+ return filepath.Join(baseDir, getLegacyExecutableName())
+}
+
+func removeLegacyExtractorBinary() error {
+ oldPath := getLegacyExtractorPath()
+ info, err := os.Stat(oldPath)
+ if err != nil || info.IsDir() {
+ return nil
+ }
+
+ if err := os.Remove(oldPath); err != nil && !os.IsNotExist(err) {
+ return fmt.Errorf("failed to remove legacy extractor binary: %v", err)
+ }
+
+ return nil
+}
diff --git a/backend/ffmpeg.go b/backend/ffmpeg.go
index ae1aa12..36f8b10 100644
--- a/backend/ffmpeg.go
+++ b/backend/ffmpeg.go
@@ -15,14 +15,12 @@ import (
"github.com/ulikunitz/xz"
)
-// FFmpeg download URLs
const (
ffmpegWindowsURL = "https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-win64-gpl.zip"
ffmpegLinuxURL = "https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz"
ffmpegMacOSURL = "https://evermeet.cx/ffmpeg/getrelease/ffmpeg/zip"
)
-// GetFFmpegPath returns the path to ffmpeg binary
func GetFFmpegPath() string {
homeDir, _ := os.UserHomeDir()
baseDir := filepath.Join(homeDir, ".twitterxmediabatchdownloader")
@@ -35,7 +33,6 @@ func GetFFmpegPath() string {
}
}
-// IsFFmpegInstalled checks if ffmpeg is available
func IsFFmpegInstalled() bool {
ffmpegPath := GetFFmpegPath()
if _, err := os.Stat(ffmpegPath); err == nil {
@@ -44,7 +41,6 @@ func IsFFmpegInstalled() bool {
return false
}
-// DownloadFFmpeg downloads ffmpeg binary for current platform
func DownloadFFmpeg(progressCallback func(downloaded, total int64)) error {
var downloadURL string
@@ -59,7 +55,6 @@ func DownloadFFmpeg(progressCallback func(downloaded, total int64)) error {
return fmt.Errorf("unsupported platform: %s", runtime.GOOS)
}
- // Create temp file for download
tempFile, err := os.CreateTemp("", "ffmpeg-*")
if err != nil {
return fmt.Errorf("failed to create temp file: %v", err)
@@ -68,7 +63,6 @@ func DownloadFFmpeg(progressCallback func(downloaded, total int64)) error {
defer os.Remove(tempPath)
defer tempFile.Close()
- // Download file
resp, err := http.Get(downloadURL)
if err != nil {
return fmt.Errorf("failed to download ffmpeg: %v", err)
@@ -79,7 +73,6 @@ func DownloadFFmpeg(progressCallback func(downloaded, total int64)) error {
return fmt.Errorf("failed to download ffmpeg: status %d", resp.StatusCode)
}
- // Copy with progress
total := resp.ContentLength
var downloaded int64
buf := make([]byte, 32*1024)
@@ -105,7 +98,6 @@ func DownloadFFmpeg(progressCallback func(downloaded, total int64)) error {
}
tempFile.Close()
- // Extract ffmpeg binary
ffmpegPath := GetFFmpegPath()
baseDir := filepath.Dir(ffmpegPath)
if err := os.MkdirAll(baseDir, 0755); err != nil {
@@ -122,7 +114,6 @@ func DownloadFFmpeg(progressCallback func(downloaded, total int64)) error {
return nil
}
-// extractFromZip extracts ffmpeg from zip archive
func extractFromZip(zipPath, destPath string) error {
r, err := zip.OpenReader(zipPath)
if err != nil {
@@ -131,7 +122,7 @@ func extractFromZip(zipPath, destPath string) error {
defer r.Close()
for _, f := range r.File {
- // Look for ffmpeg binary
+
name := filepath.Base(f.Name)
if name == "ffmpeg" || name == "ffmpeg.exe" {
rc, err := f.Open()
@@ -150,7 +141,6 @@ func extractFromZip(zipPath, destPath string) error {
return fmt.Errorf("failed to extract file: %v", err)
}
- // Make executable on Unix
if runtime.GOOS != "windows" {
os.Chmod(destPath, 0755)
}
@@ -162,7 +152,6 @@ func extractFromZip(zipPath, destPath string) error {
return fmt.Errorf("ffmpeg binary not found in archive")
}
-// extractFromTarXz extracts ffmpeg from tar.xz archive
func extractFromTarXz(tarXzPath, destPath string) error {
file, err := os.Open(tarXzPath)
if err != nil {
@@ -186,7 +175,6 @@ func extractFromTarXz(tarXzPath, destPath string) error {
return fmt.Errorf("failed to read tar: %v", err)
}
- // Look for ffmpeg binary
name := filepath.Base(header.Name)
if name == "ffmpeg" && header.Typeflag == tar.TypeReg {
out, err := os.Create(destPath)
@@ -207,9 +195,6 @@ func extractFromTarXz(tarXzPath, destPath string) error {
return fmt.Errorf("ffmpeg binary not found in archive")
}
-// ConvertMP4ToGIF converts an MP4 file to GIF using ffmpeg
-// quality: "fast" for simple conversion, "better" for optimized palette
-// resolution: "original", "high" (800px), "medium" (600px), "low" (400px)
func ConvertMP4ToGIF(inputPath, outputPath, quality, resolution string) error {
ffmpegPath := GetFFmpegPath()
@@ -220,7 +205,7 @@ func ConvertMP4ToGIF(inputPath, outputPath, quality, resolution string) error {
var args []string
if quality == "fast" {
- // Fast mode: simple conversion with resolution scaling
+
var scaleFilter string
switch resolution {
case "high":
@@ -229,7 +214,7 @@ func ConvertMP4ToGIF(inputPath, outputPath, quality, resolution string) error {
scaleFilter = "scale=600:-1"
case "low":
scaleFilter = "scale=400:-1"
- default: // original - no scaling
+ default:
scaleFilter = ""
}
@@ -250,7 +235,7 @@ func ConvertMP4ToGIF(inputPath, outputPath, quality, resolution string) error {
}
}
} else {
- // Better mode: optimized palette with dithering
+
var filter string
switch resolution {
@@ -260,11 +245,10 @@ func ConvertMP4ToGIF(inputPath, outputPath, quality, resolution string) error {
filter = "scale=600:-1:flags=lanczos,palettegen=stats_mode=full[palette];[0:v]scale=600:-1:flags=lanczos[scaled];[scaled][palette]paletteuse=dither=sierra2_4a"
case "low":
filter = "scale=400:-1:flags=lanczos,palettegen=stats_mode=full[palette];[0:v]scale=400:-1:flags=lanczos[scaled];[scaled][palette]paletteuse=dither=sierra2_4a"
- default: // original
+ default:
filter = "palettegen=stats_mode=full[palette];[0:v][palette]paletteuse=dither=sierra2_4a"
}
- // Set FPS based on resolution
fps := "15"
if resolution == "medium" {
fps = "10"
@@ -282,7 +266,7 @@ func ConvertMP4ToGIF(inputPath, outputPath, quality, resolution string) error {
}
cmd := exec.Command(ffmpegPath, args...)
- hideWindow(cmd) // Hide console window on Windows
+ hideWindow(cmd)
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("ffmpeg error: %v, output: %s", err, string(output))
@@ -291,13 +275,11 @@ func ConvertMP4ToGIF(inputPath, outputPath, quality, resolution string) error {
return nil
}
-// ConvertGIFsInFolder converts all MP4 files in gifs folder to actual GIF format
func ConvertGIFsInFolder(folderPath, quality, resolution string, deleteOriginal bool) (converted int, failed int, err error) {
if !IsFFmpegInstalled() {
return 0, 0, fmt.Errorf("ffmpeg not installed")
}
- // Clean the path to handle cross-platform path separators
cleanPath := filepath.Clean(folderPath)
gifsFolder := filepath.Join(cleanPath, "gifs")
if _, err := os.Stat(gifsFolder); os.IsNotExist(err) {
diff --git a/backend/folder.go b/backend/folder.go
index 29d66a8..f8608d6 100644
--- a/backend/folder.go
+++ b/backend/folder.go
@@ -15,9 +15,9 @@ func OpenFolderInExplorer(path string) error {
switch runtime.GOOS {
case "windows":
- // Use cmd /c start to open explorer - more reliable
+
cmd = exec.Command("cmd", "/c", "start", "", path)
- case "darwin": // macOS
+ case "darwin":
cmd = exec.Command("open", path)
case "linux":
cmd = exec.Command("xdg-open", path)
@@ -25,12 +25,12 @@ func OpenFolderInExplorer(path string) error {
cmd = exec.Command("xdg-open", path)
}
- hideWindow(cmd) // Hide console window on Windows
+ hideWindow(cmd)
return cmd.Run()
}
func SelectFolderDialog(ctx context.Context, defaultPath string) (string, error) {
- // If defaultPath is empty, use default download path
+
if defaultPath == "" {
defaultPath = GetDefaultDownloadPath()
}
@@ -45,7 +45,6 @@ func SelectFolderDialog(ctx context.Context, defaultPath string) (string, error)
return "", err
}
- // If user cancelled, selectedPath will be empty
if selectedPath == "" {
return "", nil
}
@@ -53,7 +52,6 @@ func SelectFolderDialog(ctx context.Context, defaultPath string) (string, error)
return selectedPath, nil
}
-// CheckFolderExists checks if a folder exists at the given path
func CheckFolderExists(basePath, username string) bool {
folderPath := filepath.Join(basePath, username)
info, err := os.Stat(folderPath)
@@ -63,7 +61,6 @@ func CheckFolderExists(basePath, username string) bool {
return info.IsDir()
}
-// CheckGifsFolderExists checks if a gifs subfolder exists for the given username
func CheckGifsFolderExists(basePath, username string) bool {
gifsPath := filepath.Join(basePath, username, "gifs")
info, err := os.Stat(gifsPath)
@@ -73,7 +70,6 @@ func CheckGifsFolderExists(basePath, username string) bool {
return info.IsDir()
}
-// CheckGifsFolderHasMP4 checks if the gifs folder has any MP4 files to convert
func CheckGifsFolderHasMP4(basePath, username string) bool {
gifsPath := filepath.Join(basePath, username, "gifs")
@@ -93,12 +89,10 @@ func CheckGifsFolderHasMP4(basePath, username string) bool {
return false
}
-// GetFolderPath returns the full path for a username folder
func GetFolderPath(basePath, username string) string {
return filepath.Join(basePath, username)
}
-// GetGifsFolderPath returns the full path for a username's gifs folder
func GetGifsFolderPath(basePath, username string) string {
return filepath.Join(basePath, username, "gifs")
}
diff --git a/backend/metadata.go b/backend/metadata.go
index 7c42eab..bf323bd 100644
--- a/backend/metadata.go
+++ b/backend/metadata.go
@@ -11,11 +11,6 @@ import (
"strings"
)
-// ExtractOriginalFilename extracts the original filename from Twitter media URL
-// Supports 3 formats:
-// 1. animated_gif: https://video.twimg.com/tweet_video/GzstJyDbIAAPuX9.mp4 -> GzstJyDbIAAPuX9
-// 2. photo: https://pbs.twimg.com/media/GynjhU0bYAAA_I4?format=jpg -> GynjhU0bYAAA_I4
-// 3. video: https://video.twimg.com/amplify_video/.../Jpo0cDa7aNtjSv7o.mp4 -> Jpo0cDa7aNtjSv7o
func ExtractOriginalFilename(mediaURL string) string {
parsedURL, err := url.Parse(mediaURL)
if err != nil {
@@ -25,16 +20,15 @@ func ExtractOriginalFilename(mediaURL string) string {
path := parsedURL.Path
parts := strings.Split(path, "/")
- // 1. animated_gif: video.twimg.com/tweet_video/XXX.mp4
if strings.Contains(mediaURL, "/tweet_video/") {
for i, part := range parts {
if part == "tweet_video" && i+1 < len(parts) {
filename := parts[i+1]
- // Remove extension
+
if idx := strings.LastIndex(filename, "."); idx > 0 {
filename = filename[:idx]
}
- // Validate alphanumeric + underscore (Twitter filenames can contain _)
+
if matched, _ := regexp.MatchString(`^[A-Za-z0-9_]+$`, filename); matched && len(filename) > 0 {
return filename
}
@@ -42,32 +36,28 @@ func ExtractOriginalFilename(mediaURL string) string {
}
}
- // 2. photo: pbs.twimg.com/media/XXX
if strings.Contains(mediaURL, "pbs.twimg.com/media/") {
if len(parts) > 0 {
filename := parts[len(parts)-1]
- // Remove extension if present
+
if idx := strings.LastIndex(filename, "."); idx > 0 {
filename = filename[:idx]
}
- // Validate alphanumeric + underscore (Twitter filenames can contain _)
+
if matched, _ := regexp.MatchString(`^[A-Za-z0-9_]+$`, filename); matched && len(filename) > 0 {
return filename
}
}
}
- // 3. video: video.twimg.com/amplify_video/.../XXX.mp4 or video.twimg.com/ext_tw_video/.../XXX.mp4
- // Filename is the last segment (before extension)
- // Note: Skip if already handled by tweet_video above
if strings.Contains(mediaURL, "video.twimg.com") && !strings.Contains(mediaURL, "/tweet_video/") {
if len(parts) > 0 {
filename := parts[len(parts)-1]
- // Remove extension
+
if idx := strings.LastIndex(filename, "."); idx > 0 {
filename = filename[:idx]
}
- // Validate: alphanumeric + underscore with at least one letter (to exclude pure numeric IDs)
+
if matched, _ := regexp.MatchString(`^[A-Za-z0-9_]+$`, filename); matched {
hasLetter, _ := regexp.MatchString(`[A-Za-z]`, filename)
if hasLetter && len(filename) >= 8 {
@@ -80,39 +70,30 @@ func ExtractOriginalFilename(mediaURL string) string {
return ""
}
-// EmbedMetadata embeds metadata into a media file
-// Only supports JPG (images) and MP4 (videos)
func EmbedMetadata(filePath string, tweetContent string, tweetURL string, originalFilename string) error {
ext := strings.ToLower(filepath.Ext(filePath))
-
+
switch ext {
case ".jpg", ".jpeg":
return embedImageMetadata(filePath, tweetContent, tweetURL, originalFilename)
case ".mp4":
return embedVideoMetadata(filePath, tweetContent, tweetURL, originalFilename)
default:
- // For unsupported formats, skip metadata embedding
+
return nil
}
}
-// embedImageMetadata embeds metadata into image files using exiftool or similar
-// Since we don't want to add heavy dependencies, we'll use a simple approach:
-// For JPEG: We can use exiftool if available, or skip if not
-// For PNG: Limited support, skip for now
func embedImageMetadata(filePath string, tweetContent string, tweetURL string, originalFilename string) error {
- // Try to use exiftool if available (common tool for metadata)
+
exiftoolPath := findExifTool()
if exiftoolPath == "" {
- // exiftool not found, skip metadata embedding for images
- // This is acceptable as it's an optional feature
+
return nil
}
- // Build metadata comment
metadataComment := buildMetadataComment(tweetContent, tweetURL, originalFilename)
- // Use exiftool to add comment only (URL | filename)
args := []string{
"-overwrite_original",
"-Comment=" + metadataComment,
@@ -123,32 +104,29 @@ func embedImageMetadata(filePath string, tweetContent string, tweetURL string, o
hideWindow(cmd)
output, err := cmd.CombinedOutput()
if err != nil {
- // Non-fatal: if exiftool fails, just skip metadata
+
return fmt.Errorf("exiftool error (non-fatal): %v, output: %s", err, string(output))
}
- _ = output // Suppress unused variable warning
+ _ = output
return nil
}
-// embedVideoMetadata embeds metadata into video/GIF files using ExifTool
func embedVideoMetadata(filePath string, tweetContent string, tweetURL string, originalFilename string) error {
- // Use ExifTool for video metadata (works well for MP4)
+
exiftoolPath := findExifTool()
if exiftoolPath == "" {
- // ExifTool not available, skip metadata embedding (non-fatal)
+
return nil
}
return embedVideoMetadataWithExifTool(exiftoolPath, filePath, tweetContent, tweetURL, originalFilename)
}
-// embedVideoMetadataWithExifTool embeds metadata using ExifTool (preferred for MP4)
func embedVideoMetadataWithExifTool(exiftoolPath string, filePath string, tweetContent string, tweetURL string, originalFilename string) error {
- // Build metadata comment
+
metadataComment := buildMetadataComment(tweetContent, tweetURL, originalFilename)
- // Use exiftool to add comment only (URL | filename)
args := []string{
"-overwrite_original",
"-Comment=" + metadataComment,
@@ -159,42 +137,37 @@ func embedVideoMetadataWithExifTool(exiftoolPath string, filePath string, tweetC
hideWindow(cmd)
output, err := cmd.CombinedOutput()
if err != nil {
- // Non-fatal: if exiftool fails, skip metadata (file still downloaded)
+
return fmt.Errorf("exiftool error (non-fatal): %v, output: %s", err, string(output))
}
- _ = output // Suppress unused variable warning
+ _ = output
return nil
}
-
-// buildMetadataComment builds a formatted metadata comment string
func buildMetadataComment(tweetContent string, tweetURL string, originalFilename string) string {
var parts []string
-
+
if tweetURL != "" {
parts = append(parts, tweetURL)
}
if originalFilename != "" {
parts = append(parts, originalFilename)
}
-
- // If no original filename, just return URL (don't add empty part)
+
if len(parts) == 0 {
return ""
}
-
+
return strings.Join(parts, " | ")
}
-// findExifTool searches for exiftool, prioritizing the installed version in .twitterxmediabatchdownloader
func findExifTool() string {
- // First, check if exiftool is installed in .twitterxmediabatchdownloader
+
if IsExifToolInstalled() {
return GetExifToolPath()
}
- // Fallback: check common system locations
commonPaths := []string{
"exiftool",
"/usr/bin/exiftool",
@@ -205,23 +178,21 @@ func findExifTool() string {
for _, path := range commonPaths {
if runtime.GOOS == "windows" {
- // On Windows, try .exe extension
+
if _, err := exec.LookPath(path); err == nil {
return path
}
} else {
- // On Unix, check if executable
+
if _, err := os.Stat(path); err == nil {
return path
}
}
}
- // Try to find in PATH
if path, err := exec.LookPath("exiftool"); err == nil {
return path
}
return ""
}
-
diff --git a/backend/twitter.go b/backend/twitter.go
index 6ef995e..aa2a352 100644
--- a/backend/twitter.go
+++ b/backend/twitter.go
@@ -1,8 +1,6 @@
package backend
import (
- "crypto/sha256"
- "encoding/hex"
"encoding/json"
"fmt"
"net/url"
@@ -13,38 +11,47 @@ import (
"strings"
)
-// getExecutableName returns the appropriate executable name for the current OS
func getExecutableName() string {
+ if runtime.GOOS == "windows" {
+ return "xtractor.exe"
+ }
+ return "xtractor"
+}
+
+func getLegacyExecutableName() string {
if runtime.GOOS == "windows" {
return "extractor.exe"
}
return "extractor"
}
-// KillAllExtractorProcesses kills all running extractor processes
-// This is useful for cleanup when starting fresh or when user stops fetch
func KillAllExtractorProcesses() {
- exeName := getExecutableName()
+ names := []string{getExecutableName(), getLegacyExecutableName()}
+ seen := make(map[string]bool)
- var cmd *exec.Cmd
- if runtime.GOOS == "windows" {
- // Use taskkill on Windows
- cmd = exec.Command("taskkill", "/F", "/IM", exeName)
- } else {
- // Use pkill on Unix
- cmd = exec.Command("pkill", "-f", exeName)
+ for _, exeName := range names {
+ if seen[exeName] {
+ continue
+ }
+ seen[exeName] = true
+
+ var cmd *exec.Cmd
+ if runtime.GOOS == "windows" {
+
+ cmd = exec.Command("taskkill", "/F", "/IM", exeName)
+ } else {
+
+ cmd = exec.Command("pkill", "-f", exeName)
+ }
+
+ hideWindow(cmd)
+ cmd.CombinedOutput()
}
-
- hideWindow(cmd)
- cmd.CombinedOutput() // Ignore errors - it's okay if no processes found
}
-// parseExtractorError parses the extractor output and returns a user-friendly error message
-// while preserving the original error from gallery-dl
func parseExtractorError(output string, username string) string {
outputLower := strings.ToLower(output)
- // Extract the actual error line from gallery-dl output
lines := strings.Split(output, "\n")
var errorLine string
for _, line := range lines {
@@ -58,12 +65,10 @@ func parseExtractorError(output string, username string) string {
errorLine = strings.TrimSpace(output)
}
- // Truncate if too long
if len(errorLine) > 300 {
errorLine = errorLine[:300] + "..."
}
- // Add context hint based on error type, but keep original message
var hint string
if strings.Contains(outputLower, "unable to retrieve tweets from this timeline") {
hint = " [Hint: End of timeline reached or rate limited - data already fetched has been saved]"
@@ -80,23 +85,20 @@ func parseExtractorError(output string, username string) string {
return errorLine + hint
}
-// TweetIDString is a custom type that unmarshals int64 but marshals as string
type TweetIDString int64
-// MarshalJSON converts TweetIDString to JSON string to preserve precision in JavaScript
func (t TweetIDString) MarshalJSON() ([]byte, error) {
return []byte(fmt.Sprintf(`"%d"`, t)), nil
}
-// UnmarshalJSON accepts both number and string from JSON
func (t *TweetIDString) UnmarshalJSON(data []byte) error {
- // Try to unmarshal as number first (from extractor)
+
var num int64
if err := json.Unmarshal(data, &num); err == nil {
*t = TweetIDString(num)
return nil
}
- // Try as string (for future compatibility)
+
var str string
if err := json.Unmarshal(data, &str); err == nil {
parsed, err := fmt.Sscanf(str, "%d", &num)
@@ -109,14 +111,12 @@ func (t *TweetIDString) UnmarshalJSON(data []byte) error {
return fmt.Errorf("tweet_id must be number or string")
}
-// Author represents tweet author information from extractor
type Author struct {
ID int64 `json:"id"`
Name string `json:"name"`
Nick string `json:"nick"`
}
-// UserInfo represents full user information from extractor
type UserInfo struct {
ID int64 `json:"id"`
Name string `json:"name"`
@@ -137,7 +137,6 @@ type UserInfo struct {
URL string `json:"url"`
}
-// CLIMediaItem represents a single media entry from extractor CLI
type CLIMediaItem struct {
URL string `json:"url"`
TweetID TweetIDString `json:"tweet_id"`
@@ -165,7 +164,6 @@ type CLIMediaItem struct {
Sensitive bool `json:"sensitive"`
}
-// TweetMetadata represents tweet metadata from extractor
type TweetMetadata struct {
TweetID TweetIDString `json:"tweet_id"`
RetweetID TweetIDString `json:"retweet_id,omitempty"`
@@ -186,38 +184,35 @@ type TweetMetadata struct {
Sensitive bool `json:"sensitive,omitempty"`
}
-// CLIResponse represents the raw response from extractor CLI
type CLIResponse struct {
Media []CLIMediaItem `json:"media"`
Metadata []TweetMetadata `json:"metadata"`
- Cursor string `json:"cursor,omitempty"` // Cursor for resume
- Total int `json:"total,omitempty"` // Total items fetched
- Completed bool `json:"completed,omitempty"` // True if all fetched
+ Cursor string `json:"cursor,omitempty"`
+ Total int `json:"total,omitempty"`
+ Completed bool `json:"completed,omitempty"`
}
-// TimelineEntry represents a single media entry for frontend (converted from MediaItem)
type TimelineEntry struct {
- URL string `json:"url"`
- Date string `json:"date"`
- TweetID TweetIDString `json:"tweet_id"`
- Type string `json:"type"`
- IsRetweet bool `json:"is_retweet"`
- Extension string `json:"extension"`
- Width int `json:"width"`
- Height int `json:"height"`
- Content string `json:"content,omitempty"`
- ViewCount int `json:"view_count,omitempty"`
- BookmarkCount int `json:"bookmark_count,omitempty"`
- FavoriteCount int `json:"favorite_count,omitempty"`
- RetweetCount int `json:"retweet_count,omitempty"`
- ReplyCount int `json:"reply_count,omitempty"`
- Source string `json:"source,omitempty"`
- Verified bool `json:"verified,omitempty"`
- OriginalFilename string `json:"original_filename,omitempty"` // Original filename from API
- AuthorUsername string `json:"author_username,omitempty"` // Username of tweet author (for bookmarks and likes)
+ URL string `json:"url"`
+ Date string `json:"date"`
+ TweetID TweetIDString `json:"tweet_id"`
+ Type string `json:"type"`
+ IsRetweet bool `json:"is_retweet"`
+ Extension string `json:"extension"`
+ Width int `json:"width"`
+ Height int `json:"height"`
+ Content string `json:"content,omitempty"`
+ ViewCount int `json:"view_count,omitempty"`
+ BookmarkCount int `json:"bookmark_count,omitempty"`
+ FavoriteCount int `json:"favorite_count,omitempty"`
+ RetweetCount int `json:"retweet_count,omitempty"`
+ ReplyCount int `json:"reply_count,omitempty"`
+ Source string `json:"source,omitempty"`
+ Verified bool `json:"verified,omitempty"`
+ OriginalFilename string `json:"original_filename,omitempty"`
+ AuthorUsername string `json:"author_username,omitempty"`
}
-// AccountInfo represents Twitter account information (derived from metadata)
type AccountInfo struct {
Name string `json:"name"`
Nick string `json:"nick"`
@@ -228,65 +223,58 @@ type AccountInfo struct {
StatusesCount int `json:"statuses_count"`
}
-// ExtractMetadata represents extraction metadata
type ExtractMetadata struct {
NewEntries int `json:"new_entries"`
Page int `json:"page"`
BatchSize int `json:"batch_size"`
HasMore bool `json:"has_more"`
- Cursor string `json:"cursor,omitempty"` // Cursor for resume capability
- Completed bool `json:"completed,omitempty"` // True if all media fetched
+ Cursor string `json:"cursor,omitempty"`
+ Completed bool `json:"completed,omitempty"`
}
-// TwitterResponse represents the full response for frontend
type TwitterResponse struct {
AccountInfo AccountInfo `json:"account_info"`
TotalURLs int `json:"total_urls"`
Timeline []TimelineEntry `json:"timeline"`
Metadata ExtractMetadata `json:"metadata"`
- Cursor string `json:"cursor,omitempty"` // Cursor for next fetch
- Completed bool `json:"completed,omitempty"` // True if fetch completed
+ Cursor string `json:"cursor,omitempty"`
+ Completed bool `json:"completed,omitempty"`
}
-// TimelineRequest represents request parameters for timeline extraction
type TimelineRequest struct {
Username string `json:"username"`
AuthToken string `json:"auth_token"`
- TimelineType string `json:"timeline_type"` // media, timeline, tweets, with_replies, likes, bookmarks
- BatchSize int `json:"batch_size"` // 0 = all
+ TimelineType string `json:"timeline_type"`
+ BatchSize int `json:"batch_size"`
Page int `json:"page"`
- MediaType string `json:"media_type"` // all, image, video, gif
+ MediaType string `json:"media_type"`
Retweets bool `json:"retweets"`
- Cursor string `json:"cursor,omitempty"` // Resume from this cursor position
+ Cursor string `json:"cursor,omitempty"`
}
-// DateRangeRequest represents request parameters for date range extraction
type DateRangeRequest struct {
Username string `json:"username"`
AuthToken string `json:"auth_token"`
- StartDate string `json:"start_date"` // YYYY-MM-DD
- EndDate string `json:"end_date"` // YYYY-MM-DD
+ StartDate string `json:"start_date"`
+ EndDate string `json:"end_date"`
MediaFilter string `json:"media_filter"`
Retweets bool `json:"retweets"`
}
-// buildTwitterURL constructs the Twitter URL based on username and timeline type
func buildTwitterURL(username, timelineType string) string {
- // Special case: bookmarks don't need username
+
if timelineType == "bookmarks" {
return "https://x.com/i/bookmarks"
}
- // Clean username - extract handle from URL if needed
username = cleanUsername(username)
- // Build URL based on timeline type
baseURL := "https://x.com/" + username
switch timelineType {
case "media":
return baseURL + "/media"
case "timeline":
- return baseURL + "/timeline" // Best for cursor support
+ return baseURL + "/timeline"
case "tweets":
return baseURL + "/tweets"
case "with_replies":
@@ -294,12 +282,10 @@ func buildTwitterURL(username, timelineType string) string {
case "likes":
return baseURL + "/likes"
default:
- return baseURL + "/timeline" // Default to timeline for reliable cursor
+ return baseURL + "/timeline"
}
}
-// cleanUsername extracts the handle from different input formats
-// Handles: @username, username, https://x.com/username, https://x.com/username/media, etc.
func cleanUsername(username string) string {
username = strings.TrimSpace(username)
username = strings.TrimPrefix(username, "@")
@@ -311,12 +297,12 @@ func cleanUsername(username string) string {
}
if u, err := url.Parse(parsed); err == nil {
segments := strings.Split(strings.Trim(u.Path, "/"), "/")
- // Skip special paths like /i/bookmarks, /search, /home, /explore
+
if len(segments) > 0 && segments[0] != "" {
firstSegment := strings.ToLower(segments[0])
- // These are not usernames
+
if firstSegment == "i" || firstSegment == "search" || firstSegment == "home" || firstSegment == "explore" || firstSegment == "settings" || firstSegment == "messages" || firstSegment == "notifications" {
- return username // Return as-is, let caller handle
+ return username
}
return segments[0]
}
@@ -326,7 +312,6 @@ func cleanUsername(username string) string {
return username
}
-// ensureURLScheme normalises URLs so the CLI accepts them
func ensureURLScheme(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
@@ -341,7 +326,6 @@ func ensureURLScheme(raw string) string {
return "https://" + strings.TrimPrefix(raw, "//")
}
-// buildSearchURL assembles an X.com search URL for date range queries
func buildSearchURL(username, startDate, endDate, mediaFilter string, includeRetweets bool) string {
trimmed := strings.TrimSpace(username)
lower := strings.ToLower(trimmed)
@@ -380,31 +364,28 @@ func buildSearchURL(username, startDate, endDate, mediaFilter string, includeRet
return fmt.Sprintf("https://x.com/search?q=%s&src=typed_query&f=live", query)
}
-// convertMetadataToTimelineEntry converts metadata-only tweets to timeline entries
func convertMetadataToTimelineEntry(meta TweetMetadata) TimelineEntry {
return TimelineEntry{
- URL: "",
- Date: meta.Date,
- TweetID: meta.TweetID,
- Type: "text",
- IsRetweet: meta.RetweetID != 0,
- Extension: "txt",
- Width: 0,
- Height: 0,
- Content: meta.Content,
- ViewCount: meta.ViewCount,
- BookmarkCount: meta.BookmarkCount,
- FavoriteCount: meta.FavoriteCount,
- RetweetCount: meta.RetweetCount,
- ReplyCount: meta.ReplyCount,
+ URL: "",
+ Date: meta.Date,
+ TweetID: meta.TweetID,
+ Type: "text",
+ IsRetweet: meta.RetweetID != 0,
+ Extension: "txt",
+ Width: 0,
+ Height: 0,
+ Content: meta.Content,
+ ViewCount: meta.ViewCount,
+ BookmarkCount: meta.BookmarkCount,
+ FavoriteCount: meta.FavoriteCount,
+ RetweetCount: meta.RetweetCount,
+ ReplyCount: meta.ReplyCount,
AuthorUsername: meta.Author.Name,
}
}
-// convertToTimelineEntry converts CLIMediaItem to TimelineEntry
func convertToTimelineEntry(media CLIMediaItem) TimelineEntry {
- // Get username from Author field (preferred for bookmarks/likes) or User field
- // Author field always contains the tweet author, while User might be the account fetching for likes
+
authorUsername := ""
if media.Author.Name != "" {
authorUsername = media.Author.Name
@@ -413,26 +394,24 @@ func convertToTimelineEntry(media CLIMediaItem) TimelineEntry {
}
entry := TimelineEntry{
- URL: media.URL,
- TweetID: media.TweetID,
- Date: media.Date,
- Extension: media.Extension,
- Width: media.Width,
- Height: media.Height,
- IsRetweet: media.RetweetID != 0,
- Content: media.Content,
- ViewCount: media.ViewCount,
- BookmarkCount: media.BookmarkCount,
- FavoriteCount: media.FavoriteCount,
- RetweetCount: media.RetweetCount,
- ReplyCount: media.ReplyCount,
- Source: media.Source,
- Verified: media.Author.Verified,
- AuthorUsername: authorUsername,
- // OriginalFilename will be extracted from URL in download.go
+ URL: media.URL,
+ TweetID: media.TweetID,
+ Date: media.Date,
+ Extension: media.Extension,
+ Width: media.Width,
+ Height: media.Height,
+ IsRetweet: media.RetweetID != 0,
+ Content: media.Content,
+ ViewCount: media.ViewCount,
+ BookmarkCount: media.BookmarkCount,
+ FavoriteCount: media.FavoriteCount,
+ RetweetCount: media.RetweetCount,
+ ReplyCount: media.ReplyCount,
+ Source: media.Source,
+ Verified: media.Author.Verified,
+ AuthorUsername: authorUsername,
}
- // Determine type - media item already has type from CLI
if media.Type != "" {
entry.Type = media.Type
} else {
@@ -449,119 +428,52 @@ func convertToTimelineEntry(media CLIMediaItem) TimelineEntry {
return entry
}
-// getExtractorPath returns the path to extractor binary
-// Binary is stored in ~/.twitterxmediabatchdownloader/ (same as ffmpeg and database)
func getExtractorPath() string {
homeDir, _ := os.UserHomeDir()
baseDir := filepath.Join(homeDir, ".twitterxmediabatchdownloader")
return filepath.Join(baseDir, getExecutableName())
}
-// getHashFilePath returns the path to the hash file for version checking
-func getHashFilePath() string {
- homeDir, _ := os.UserHomeDir()
- baseDir := filepath.Join(homeDir, ".twitterxmediabatchdownloader")
- return filepath.Join(baseDir, "extractor.sha256")
-}
-
-// calculateHash calculates SHA256 hash of data
-func calculateHash(data []byte) string {
- hash := sha256.Sum256(data)
- return hex.EncodeToString(hash[:])
-}
-
-// ensureExtractor ensures the extractor binary exists
-// Extracts from embedded binary if not present or if hash differs (update)
-func ensureExtractor() (string, error) {
- exePath := getExtractorPath()
- hashPath := getHashFilePath()
- baseDir := filepath.Dir(exePath)
-
- // Create directory if not exists
- if err := os.MkdirAll(baseDir, 0755); err != nil {
- return "", fmt.Errorf("failed to create directory: %v", err)
- }
-
- // Calculate hash of embedded binary
- embeddedHash := calculateHash(extractorBin)
-
- // Check if binary and hash file exist
- if _, err := os.Stat(exePath); err == nil {
- // Binary exists - check hash
- if storedHash, err := os.ReadFile(hashPath); err == nil {
- if string(storedHash) == embeddedHash {
- return exePath, nil // Already extracted and up to date
- }
- }
- // Hash differs or missing - need to update
- os.Remove(exePath)
- }
-
- // Extract binary
- if err := os.WriteFile(exePath, extractorBin, 0755); err != nil {
- return "", fmt.Errorf("failed to write extractor: %v", err)
- }
-
- // Save hash for future comparison
- if err := os.WriteFile(hashPath, []byte(embeddedHash), 0644); err != nil {
- // Non-fatal - binary still works, just won't skip next time
- fmt.Printf("Warning: failed to save hash file: %v\n", err)
- }
-
- return exePath, nil
-}
-
-// ExtractTimeline extracts media from user timeline using the new CLI
func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) {
- // Get or extract extractor binary (persistent, not temp)
- exePath, err := ensureExtractor()
+
+ exePath, err := requireExtractorPath()
if err != nil {
return nil, err
}
- // Determine the right endpoint based on what user wants:
- // - Media (all/image/video/gif): Use /media endpoint - fastest and most reliable
- // - Text tweets: Use /tweets endpoint with --text-tweets
- // - With retweets: Use /tweets endpoint (retweets not available on /media)
isTextOnly := req.MediaType == "text"
wantsRetweets := req.Retweets
timelineType := req.TimelineType
if timelineType == "" {
if isTextOnly {
- // Text-only tweets need /tweets endpoint
+
timelineType = "tweets"
} else if wantsRetweets {
- // Retweets need /tweets endpoint (not available on /media)
+
timelineType = "tweets"
} else {
- // Default: /media endpoint - fastest for media-only fetch
+
timelineType = "media"
}
}
url := buildTwitterURL(req.Username, timelineType)
- // Build command arguments for new CLI format
- // Format: extractor.exe URL --auth-token TOKEN --json [options]
args := []string{url}
- // Add auth token
if req.AuthToken != "" {
args = append(args, "--auth-token", req.AuthToken)
} else {
args = append(args, "--guest")
}
- // Always request JSON output with metadata
args = append(args, "--json", "--metadata")
- // Add limit if specified
if req.BatchSize > 0 {
args = append(args, "--limit", fmt.Sprintf("%d", req.BatchSize))
}
- // Handle retweets - only relevant when using /tweets endpoint
if timelineType == "tweets" || timelineType == "timeline" {
if req.Retweets {
args = append(args, "--retweets", "include")
@@ -570,12 +482,10 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) {
}
}
- // Only add --text-tweets when explicitly requesting text content
if isTextOnly {
args = append(args, "--text-tweets")
}
- // Handle media type filter using --type parameter
if req.MediaType != "" && req.MediaType != "all" && !isTextOnly {
switch req.MediaType {
case "image":
@@ -587,21 +497,18 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) {
}
}
- // Add cursor for resume capability
if req.Cursor != "" {
args = append(args, "--cursor", req.Cursor)
}
- // Execute command with UTF-8 encoding
cmd := exec.Command(exePath, args...)
cmd.Env = append(os.Environ(),
"PYTHONIOENCODING=utf-8",
"PYTHONUTF8=1",
)
- hideWindow(cmd) // Hide console window on Windows
+ hideWindow(cmd)
output, err := cmd.CombinedOutput()
- // Ensure process is killed after completion
if cmd.Process != nil {
cmd.Process.Kill()
}
@@ -612,36 +519,31 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) {
return nil, fmt.Errorf("%s", errorMsg)
}
- // Find JSON in output (skip any info messages)
jsonStr := extractJSON(string(output))
if jsonStr == "" {
outputStr := string(output)
if strings.TrimSpace(outputStr) == "" {
- return nil, fmt.Errorf("empty_response: Extractor returned no data. The timeline may be empty or inaccessible")
+ return nil, fmt.Errorf("empty_response: Xtractor returned no data. The timeline may be empty or inaccessible")
}
- return nil, fmt.Errorf("parse_error: Could not parse extractor output. Raw output: %s", outputStr)
+ return nil, fmt.Errorf("parse_error: Could not parse xtractor output. Raw output: %s", outputStr)
}
- // Parse CLI response
var cliResponse CLIResponse
if err := json.Unmarshal([]byte(jsonStr), &cliResponse); err != nil {
return nil, fmt.Errorf("json_error: Failed to parse JSON response: %v", err)
}
- // Convert to frontend format
var timeline []TimelineEntry
accountInfo := AccountInfo{
Name: req.Username,
Nick: req.Username,
}
- // Build a set of tweet IDs that have media
mediaTweetIDs := make(map[int64]bool)
for _, media := range cliResponse.Media {
mediaTweetIDs[int64(media.TweetID)] = true
}
- // For bookmarks and likes, keep name as "bookmarks"/"likes" (not from author tweet)
isBookmarks := req.TimelineType == "bookmarks"
isLikes := req.TimelineType == "likes"
if isBookmarks {
@@ -653,7 +555,7 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) {
}
if isTextOnly {
- // Text-only mode: get tweets from metadata that don't have media
+
timeline = make([]TimelineEntry, 0)
for _, meta := range cliResponse.Metadata {
if !mediaTweetIDs[int64(meta.TweetID)] {
@@ -661,7 +563,6 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) {
}
}
- // Get account info from first media item if available, otherwise from metadata
if !isBookmarks && !isLikes {
if len(cliResponse.Media) > 0 {
user := cliResponse.Media[0].User
@@ -678,7 +579,7 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) {
accountInfo.Nick = firstMeta.Author.Nick
}
} else {
- // For bookmarks and likes, get other info from first media item if available
+
if len(cliResponse.Media) > 0 {
user := cliResponse.Media[0].User
accountInfo.Date = user.Date
@@ -689,15 +590,13 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) {
}
}
} else if len(cliResponse.Media) > 0 {
- // Normal media items only (text tweets handled separately with "text" media type)
+
timeline = make([]TimelineEntry, 0, len(cliResponse.Media))
- // Add media items
for _, media := range cliResponse.Media {
timeline = append(timeline, convertToTimelineEntry(media))
}
- // Get account info from first media item
user := cliResponse.Media[0].User
if !isBookmarks && !isLikes {
accountInfo.Name = user.Name
@@ -709,29 +608,29 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) {
accountInfo.ProfileImage = user.ProfileImage
accountInfo.StatusesCount = user.StatusesCount
} else if len(cliResponse.Metadata) > 0 {
- // Fallback: Text-only tweets (no media) - convert metadata to timeline entries
+
timeline = make([]TimelineEntry, 0, len(cliResponse.Metadata))
for _, meta := range cliResponse.Metadata {
entry := TimelineEntry{
- URL: "", // No media URL for text tweets
- TweetID: meta.TweetID,
- Date: meta.Date,
- Type: "text",
- IsRetweet: meta.RetweetID != 0,
- Extension: "txt",
- Width: 0,
- Height: 0,
- Content: meta.Content,
- ViewCount: meta.ViewCount,
- BookmarkCount: meta.BookmarkCount,
- FavoriteCount: meta.FavoriteCount,
- RetweetCount: meta.RetweetCount,
- ReplyCount: meta.ReplyCount,
+ URL: "",
+ TweetID: meta.TweetID,
+ Date: meta.Date,
+ Type: "text",
+ IsRetweet: meta.RetweetID != 0,
+ Extension: "txt",
+ Width: 0,
+ Height: 0,
+ Content: meta.Content,
+ ViewCount: meta.ViewCount,
+ BookmarkCount: meta.BookmarkCount,
+ FavoriteCount: meta.FavoriteCount,
+ RetweetCount: meta.RetweetCount,
+ ReplyCount: meta.ReplyCount,
AuthorUsername: meta.Author.Name,
}
timeline = append(timeline, entry)
}
- // Get account info from first metadata
+
if !isBookmarks && !isLikes {
firstMeta := cliResponse.Metadata[0]
accountInfo.Name = firstMeta.Author.Name
@@ -739,7 +638,6 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) {
}
}
- // Determine if there's more data to fetch
hasMore := cliResponse.Cursor != "" && !cliResponse.Completed
response := &TwitterResponse{
@@ -761,10 +659,9 @@ func ExtractTimeline(req TimelineRequest) (*TwitterResponse, error) {
return response, nil
}
-// ExtractDateRange extracts media based on date range using the new CLI
func ExtractDateRange(req DateRangeRequest) (*TwitterResponse, error) {
- // Get or extract extractor binary (persistent, not temp)
- exePath, err := ensureExtractor()
+
+ exePath, err := requireExtractorPath()
if err != nil {
return nil, err
}
@@ -772,17 +669,14 @@ func ExtractDateRange(req DateRangeRequest) (*TwitterResponse, error) {
mediaFilter := strings.ToLower(strings.TrimSpace(req.MediaFilter))
url := buildSearchURL(req.Username, req.StartDate, req.EndDate, mediaFilter, req.Retweets)
- // Build command arguments
args := []string{url}
- // Add auth token
if req.AuthToken != "" {
args = append(args, "--auth-token", req.AuthToken)
} else {
args = append(args, "--guest")
}
- // Always request JSON output with metadata
args = append(args, "--json", "--metadata")
if req.Retweets {
@@ -796,7 +690,6 @@ func ExtractDateRange(req DateRangeRequest) (*TwitterResponse, error) {
args = append(args, "--text-tweets")
}
- // Execute command with UTF-8 encoding
cmd := exec.Command(exePath, args...)
cmd.Env = append(os.Environ(),
"PYTHONIOENCODING=utf-8",
@@ -805,7 +698,6 @@ func ExtractDateRange(req DateRangeRequest) (*TwitterResponse, error) {
hideWindow(cmd)
output, err := cmd.CombinedOutput()
- // Ensure process is killed after completion
if cmd.Process != nil {
cmd.Process.Kill()
}
@@ -816,23 +708,20 @@ func ExtractDateRange(req DateRangeRequest) (*TwitterResponse, error) {
return nil, fmt.Errorf("%s", errorMsg)
}
- // Find JSON in output (skip any info messages)
jsonStr := extractJSON(string(output))
if jsonStr == "" {
outputStr := string(output)
if strings.TrimSpace(outputStr) == "" {
- return nil, fmt.Errorf("empty_response: Extractor returned no data. The timeline may be empty or inaccessible")
+ return nil, fmt.Errorf("empty_response: Xtractor returned no data. The timeline may be empty or inaccessible")
}
- return nil, fmt.Errorf("parse_error: Could not parse extractor output. Raw output: %s", outputStr)
+ return nil, fmt.Errorf("parse_error: Could not parse xtractor output. Raw output: %s", outputStr)
}
- // Parse CLI response
var cliResponse CLIResponse
if err := json.Unmarshal([]byte(jsonStr), &cliResponse); err != nil {
return nil, fmt.Errorf("json_error: Failed to parse JSON response: %v", err)
}
- // Convert to frontend format
mediaTweetIDs := make(map[int64]bool)
for _, media := range cliResponse.Media {
mediaTweetIDs[int64(media.TweetID)] = true
@@ -851,7 +740,6 @@ func ExtractDateRange(req DateRangeRequest) (*TwitterResponse, error) {
}
}
- // Build account info from first media item (has full user info)
accountInfo := AccountInfo{
Name: req.Username,
Nick: req.Username,
@@ -871,7 +759,6 @@ func ExtractDateRange(req DateRangeRequest) (*TwitterResponse, error) {
accountInfo.Nick = firstMeta.Author.Nick
}
- // Determine if there's more data to fetch
hasMore := cliResponse.Cursor != "" && !cliResponse.Completed
response := &TwitterResponse{
@@ -893,15 +780,13 @@ func ExtractDateRange(req DateRangeRequest) (*TwitterResponse, error) {
return response, nil
}
-// extractJSON finds and extracts JSON object from output string
func extractJSON(output string) string {
- // Find the start of JSON object
+
start := strings.Index(output, "{")
if start == -1 {
return ""
}
- // Find the matching closing brace
depth := 0
for i := start; i < len(output); i++ {
if output[i] == '{' {
diff --git a/frontend/eslint.config.js b/frontend/eslint.config.js
index 5e6b472..13d2e6b 100644
--- a/frontend/eslint.config.js
+++ b/frontend/eslint.config.js
@@ -1,23 +1,22 @@
-import js from '@eslint/js'
-import globals from 'globals'
-import reactHooks from 'eslint-plugin-react-hooks'
-import reactRefresh from 'eslint-plugin-react-refresh'
-import tseslint from 'typescript-eslint'
-import { defineConfig, globalIgnores } from 'eslint/config'
-
+import js from '@eslint/js';
+import globals from 'globals';
+import reactHooks from 'eslint-plugin-react-hooks';
+import reactRefresh from 'eslint-plugin-react-refresh';
+import tseslint from 'typescript-eslint';
+import { defineConfig, globalIgnores } from 'eslint/config';
export default defineConfig([
- globalIgnores(['dist']),
- {
- files: ['**/*.{ts,tsx}'],
- extends: [
- js.configs.recommended,
- tseslint.configs.recommended,
- reactHooks.configs.flat.recommended,
- reactRefresh.configs.vite,
- ],
- languageOptions: {
- ecmaVersion: 2020,
- globals: globals.browser,
+ globalIgnores(['dist']),
+ {
+ files: ['**/*.{ts,tsx}'],
+ extends: [
+ js.configs.recommended,
+ tseslint.configs.recommended,
+ reactHooks.configs.flat.recommended,
+ reactRefresh.configs.vite,
+ ],
+ languageOptions: {
+ ecmaVersion: 2020,
+ globals: globals.browser,
+ },
},
- },
-])
+]);
diff --git a/frontend/package.json.md5 b/frontend/package.json.md5
index ac6d48d..2d07d73 100644
--- a/frontend/package.json.md5
+++ b/frontend/package.json.md5
@@ -1 +1 @@
-93300717a79529efe3943af0fda63408
\ No newline at end of file
+cb4545ab700ecc9d641ab1be76b622f1
\ No newline at end of file
diff --git a/frontend/scripts/generate-icon.js b/frontend/scripts/generate-icon.js
index b975dd8..02eec68 100644
--- a/frontend/scripts/generate-icon.js
+++ b/frontend/scripts/generate-icon.js
@@ -2,32 +2,23 @@ import sharp from 'sharp';
import { readFileSync, mkdirSync } from 'fs';
import { dirname, join } from 'path';
import { fileURLToPath } from 'url';
-
const __dirname = dirname(fileURLToPath(import.meta.url));
const rootDir = join(__dirname, '..', '..');
-
const svgPath = join(rootDir, 'frontend', 'public', 'icon.svg');
const outputPath = join(rootDir, 'build', 'appicon.png');
-
async function generateIcon() {
- try {
- // Ensure build directory exists
- mkdirSync(join(rootDir, 'build'), { recursive: true });
-
- // Read SVG
- const svgBuffer = readFileSync(svgPath);
-
- // Convert SVG to PNG (1024x1024 for Wails)
- await sharp(svgBuffer)
- .resize(1024, 1024)
- .png()
- .toFile(outputPath);
-
- console.log('✓ Icon generated:', outputPath);
- } catch (error) {
- console.error('✗ Failed to generate icon:', error.message);
- process.exit(1);
- }
+ try {
+ mkdirSync(join(rootDir, 'build'), { recursive: true });
+ const svgBuffer = readFileSync(svgPath);
+ await sharp(svgBuffer)
+ .resize(1024, 1024)
+ .png()
+ .toFile(outputPath);
+ console.log('✓ Icon generated:', outputPath);
+ }
+ catch (error) {
+ console.error('✗ Failed to generate icon:', error.message);
+ process.exit(1);
+ }
}
-
generateIcon();
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index 78fcdb0..0ce4e9e 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -4,20 +4,7 @@ import { getSettings, applyThemeMode, applyFont } from "@/lib/settings";
import { applyTheme } from "@/lib/themes";
import { toastWithSound as toast } from "@/lib/toast-with-sound";
import { logger } from "@/lib/logger";
-import {
- saveFetchState,
- getFetchState,
- clearFetchState,
- getResumableInfo,
- stateToResponse,
- mergeTimelines,
- saveCursor,
- getCursor,
- clearCursor,
- type FetchState,
-} from "@/lib/fetch-state";
-
-// Components
+import { saveFetchState, getFetchState, clearFetchState, getResumableInfo, stateToResponse, mergeTimelines, saveCursor, getCursor, clearCursor, type FetchState, } from "@/lib/fetch-state";
import { TitleBar } from "@/components/TitleBar";
import { Sidebar, type PageType } from "@/components/Sidebar";
import { Header } from "@/components/Header";
@@ -26,1803 +13,1299 @@ import { MediaList } from "@/components/MediaList";
import { DatabaseView } from "@/components/DatabaseView";
import { SettingsPage } from "@/components/SettingsPage";
import { DebugLoggerPage } from "@/components/DebugLoggerPage";
+import { DependencySetupDialog } from "@/components/DependencySetupDialog";
import type { HistoryItem } from "@/components/FetchHistory";
import type { TwitterResponse } from "@/types/api";
-
-// Wails bindings
import { ExtractTimeline, ExtractDateRange, SaveAccountToDBWithStatus, CleanupExtractorProcesses, GetAllAccountsFromDB } from "../wailsjs/go/main/App";
-
const HISTORY_KEY = "twitter_media_fetch_history";
const MAX_HISTORY = 10;
-const CURRENT_VERSION = "4.3";
-const BATCH_SIZE = 200; // Fetch in batches for progressive display and resume
-
+const CURRENT_VERSION = __APP_VERSION__;
+const BATCH_SIZE = 200;
function formatNumberWithComma(num: number): string {
- return num.toLocaleString();
+ return num.toLocaleString();
}
-
function App() {
- const [currentPage, setCurrentPage] = useState("main");
- const [username, setUsername] = useState("");
- const [loading, setLoading] = useState(false);
- const [result, setResult] = useState(null);
- const [fetchedMediaType, setFetchedMediaType] = useState("all");
- const [hasUpdate, setHasUpdate] = useState(false);
- const [releaseDate, setReleaseDate] = useState(null);
- const [fetchHistory, setFetchHistory] = useState([]);
- const [resumeInfo, setResumeInfo] = useState<{ canResume: boolean; mediaCount: number } | null>(null);
- const [elapsedTime, setElapsedTime] = useState(0);
- const [remainingTime, setRemainingTime] = useState(null);
- const [newMediaCount, setNewMediaCount] = useState(null);
- const stopFetchRef = useRef(false);
- const fetchStartTimeRef = useRef(null);
- const timeoutIntervalRef = useRef(null);
-
- // Multiple mode state
- const [fetchType, setFetchType] = useState("single");
- const [multipleAccounts, setMultipleAccounts] = useState([]);
- const [isFetchingAll, setIsFetchingAll] = useState(false);
-
- // Mode state for SearchBar
- const [searchMode, setSearchMode] = useState("public");
- const [searchPrivateType, setSearchPrivateType] = useState("bookmarks");
- const stopAllRef = useRef(false);
- const accountTimersRef = useRef