v3.8
No files matched your search
@@ -0,0 +1,14 @@
|
||||
from core.databaseManager import DatabaseManager
|
||||
from core.metadataFetcher import get_metadata, get_metadata_by_date
|
||||
from core.downloadWorker import DownloadWorker
|
||||
from core.metadataWorker import MetadataFetchWorker
|
||||
from core.accountModel import Account
|
||||
|
||||
__all__ = [
|
||||
'DatabaseManager',
|
||||
'get_metadata',
|
||||
'get_metadata_by_date',
|
||||
'DownloadWorker',
|
||||
'MetadataFetchWorker',
|
||||
'Account'
|
||||
]
|
||||
@@ -0,0 +1,15 @@
|
||||
from dataclasses import dataclass
|
||||
|
||||
@dataclass
|
||||
class Account:
|
||||
username: str
|
||||
nick: str
|
||||
followers: int
|
||||
following: int
|
||||
posts: int
|
||||
media_type: str
|
||||
profile_image: str = None
|
||||
media_list: list = None
|
||||
fetch_mode: str = 'all'
|
||||
fetch_timestamp: str = None
|
||||
group_id: int = None
|
||||
@@ -0,0 +1,420 @@
|
||||
import sqlite3
|
||||
import os
|
||||
import json
|
||||
import threading
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Optional, Tuple
|
||||
|
||||
class DatabaseManager:
|
||||
def __init__(self, db_path: str):
|
||||
self.db_path = db_path
|
||||
self.conn = None
|
||||
self._lock = threading.Lock()
|
||||
self.init_database()
|
||||
|
||||
def init_database(self):
|
||||
self.conn = sqlite3.connect(self.db_path, check_same_thread=False)
|
||||
self.conn.row_factory = sqlite3.Row
|
||||
|
||||
self.conn.execute('PRAGMA foreign_keys = ON')
|
||||
|
||||
cursor = self.conn.cursor()
|
||||
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS groups (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT UNIQUE NOT NULL,
|
||||
created_at TEXT
|
||||
)
|
||||
''')
|
||||
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS accounts (
|
||||
username TEXT PRIMARY KEY COLLATE NOCASE,
|
||||
nick TEXT,
|
||||
followers INTEGER,
|
||||
following INTEGER,
|
||||
posts INTEGER,
|
||||
profile_image TEXT,
|
||||
fetch_timestamp TEXT,
|
||||
media_type TEXT,
|
||||
fetch_mode TEXT,
|
||||
timeline_type TEXT DEFAULT 'media',
|
||||
group_id INTEGER,
|
||||
FOREIGN KEY (group_id) REFERENCES groups(id) ON DELETE SET NULL
|
||||
)
|
||||
''')
|
||||
|
||||
try:
|
||||
cursor.execute("SELECT group_id FROM accounts LIMIT 1")
|
||||
except sqlite3.OperationalError:
|
||||
cursor.execute("ALTER TABLE accounts ADD COLUMN group_id INTEGER")
|
||||
self.conn.commit()
|
||||
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS media (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
username TEXT COLLATE NOCASE,
|
||||
tweet_id TEXT,
|
||||
url TEXT,
|
||||
date TEXT,
|
||||
type TEXT,
|
||||
FOREIGN KEY (username) REFERENCES accounts(username) ON DELETE CASCADE,
|
||||
UNIQUE(username, url)
|
||||
)
|
||||
''')
|
||||
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS downloads (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
url TEXT,
|
||||
filepath TEXT,
|
||||
filepath_normalized TEXT COLLATE NOCASE,
|
||||
username TEXT COLLATE NOCASE,
|
||||
tweet_id TEXT,
|
||||
download_date TEXT,
|
||||
file_size INTEGER,
|
||||
status TEXT,
|
||||
UNIQUE(url, filepath_normalized)
|
||||
)
|
||||
''')
|
||||
|
||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_filepath_normalized ON downloads(filepath_normalized)')
|
||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_url ON downloads(url)')
|
||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_username ON downloads(username)')
|
||||
cursor.execute('CREATE INDEX IF NOT EXISTS idx_media_username ON media(username)')
|
||||
|
||||
self.conn.commit()
|
||||
|
||||
def save_account(self, username: str, nick: str, followers: int, following: int,
|
||||
posts: int, media_type: str, profile_image: str = None,
|
||||
fetch_mode: str = 'all', timeline_type: str = 'media',
|
||||
fetch_timestamp: str = None, group_id: int = None):
|
||||
with self._lock:
|
||||
cursor = self.conn.cursor()
|
||||
|
||||
if fetch_timestamp is None:
|
||||
fetch_timestamp = datetime.now().isoformat()
|
||||
|
||||
cursor.execute('''
|
||||
INSERT OR REPLACE INTO accounts
|
||||
(username, nick, followers, following, posts, profile_image,
|
||||
fetch_timestamp, media_type, fetch_mode, timeline_type, group_id)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
''', (username, nick, followers, following, posts, profile_image,
|
||||
fetch_timestamp, media_type, fetch_mode, timeline_type, group_id))
|
||||
|
||||
self.conn.commit()
|
||||
|
||||
def save_media_list(self, username: str, media_list: List[Dict]):
|
||||
with self._lock:
|
||||
cursor = self.conn.cursor()
|
||||
|
||||
cursor.execute('DELETE FROM media WHERE username = ?', (username,))
|
||||
|
||||
for item in media_list:
|
||||
cursor.execute('''
|
||||
INSERT OR IGNORE INTO media (username, tweet_id, url, date, type)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
''', (username, item.get('tweet_id', ''), item['url'],
|
||||
item['date'], item.get('type', 'image')))
|
||||
|
||||
self.conn.commit()
|
||||
|
||||
def get_account(self, username: str, media_type: str = 'all') -> Optional[Dict]:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute('''
|
||||
SELECT * FROM accounts
|
||||
WHERE username = ? AND media_type = ?
|
||||
''', (username, media_type))
|
||||
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
return dict(row)
|
||||
return None
|
||||
|
||||
def get_media_list(self, username: str) -> List[Dict]:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute('''
|
||||
SELECT tweet_id, url, date, type
|
||||
FROM media
|
||||
WHERE username = ?
|
||||
ORDER BY date DESC
|
||||
''', (username,))
|
||||
|
||||
return [dict(row) for row in cursor.fetchall()]
|
||||
|
||||
def get_all_accounts(self) -> List[Dict]:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute('SELECT * FROM accounts ORDER BY fetch_timestamp DESC')
|
||||
return [dict(row) for row in cursor.fetchall()]
|
||||
|
||||
def delete_account(self, username: str, media_type: str = None):
|
||||
with self._lock:
|
||||
cursor = self.conn.cursor()
|
||||
|
||||
if media_type:
|
||||
cursor.execute('DELETE FROM accounts WHERE username = ? AND media_type = ?',
|
||||
(username, media_type))
|
||||
else:
|
||||
cursor.execute('DELETE FROM accounts WHERE username = ?', (username,))
|
||||
|
||||
self.conn.commit()
|
||||
|
||||
def is_file_downloaded(self, filepath: str) -> bool:
|
||||
cursor = self.conn.cursor()
|
||||
filepath_normalized = filepath.lower()
|
||||
|
||||
cursor.execute('''
|
||||
SELECT 1 FROM downloads
|
||||
WHERE filepath_normalized = ? AND status = 'downloaded'
|
||||
LIMIT 1
|
||||
''', (filepath_normalized,))
|
||||
|
||||
return cursor.fetchone() is not None
|
||||
|
||||
def is_url_downloaded(self, url: str) -> bool:
|
||||
cursor = self.conn.cursor()
|
||||
|
||||
cursor.execute('''
|
||||
SELECT 1 FROM downloads
|
||||
WHERE url = ? AND status = 'downloaded'
|
||||
LIMIT 1
|
||||
''', (url,))
|
||||
|
||||
return cursor.fetchone() is not None
|
||||
|
||||
def record_download(self, url: str, filepath: str, username: str,
|
||||
tweet_id: str, status: str = 'downloaded', file_size: int = 0):
|
||||
with self._lock:
|
||||
cursor = self.conn.cursor()
|
||||
filepath_normalized = filepath.lower()
|
||||
download_date = datetime.now().isoformat()
|
||||
|
||||
cursor.execute('''
|
||||
INSERT OR REPLACE INTO downloads
|
||||
(url, filepath, filepath_normalized, username, tweet_id,
|
||||
download_date, file_size, status)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
''', (url, filepath, filepath_normalized, username, tweet_id,
|
||||
download_date, file_size, status))
|
||||
|
||||
self.conn.commit()
|
||||
|
||||
def get_download_stats(self, username: str = None) -> Dict:
|
||||
cursor = self.conn.cursor()
|
||||
|
||||
if username:
|
||||
cursor.execute('''
|
||||
SELECT
|
||||
COUNT(*) as total,
|
||||
SUM(CASE WHEN status = 'downloaded' THEN 1 ELSE 0 END) as downloaded,
|
||||
SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) as skipped,
|
||||
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed,
|
||||
SUM(file_size) as total_size
|
||||
FROM downloads
|
||||
WHERE username = ?
|
||||
''', (username,))
|
||||
else:
|
||||
cursor.execute('''
|
||||
SELECT
|
||||
COUNT(*) as total,
|
||||
SUM(CASE WHEN status = 'downloaded' THEN 1 ELSE 0 END) as downloaded,
|
||||
SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) as skipped,
|
||||
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed,
|
||||
SUM(file_size) as total_size
|
||||
FROM downloads
|
||||
''')
|
||||
|
||||
row = cursor.fetchone()
|
||||
return dict(row) if row else {}
|
||||
|
||||
def migrate_from_json(self, json_dir: str) -> Tuple[int, int]:
|
||||
if not os.path.exists(json_dir):
|
||||
return 0, 0
|
||||
|
||||
json_files = [f for f in os.listdir(json_dir) if f.endswith('.json')]
|
||||
accounts_migrated = 0
|
||||
media_migrated = 0
|
||||
|
||||
for json_file in json_files:
|
||||
try:
|
||||
json_path = os.path.join(json_dir, json_file)
|
||||
|
||||
with open(json_path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
media_type = data.get('media_type', 'all')
|
||||
|
||||
if media_type not in ['all', 'image', 'video', 'gif']:
|
||||
filename_parts = json_file.replace('.json', '').split('_')
|
||||
if len(filename_parts) >= 1:
|
||||
last_part = filename_parts[-1]
|
||||
if last_part in ['all', 'image', 'video', 'gif']:
|
||||
media_type = last_part
|
||||
else:
|
||||
media_type = 'all'
|
||||
else:
|
||||
media_type = 'all'
|
||||
|
||||
filename_parts = json_file.replace('.json', '').split('_')
|
||||
if len(filename_parts) >= 2:
|
||||
second_last = filename_parts[-2]
|
||||
if second_last in ['media', 'timeline', 'tweets', 'with']:
|
||||
timeline_type = second_last
|
||||
else:
|
||||
timeline_type = 'media'
|
||||
else:
|
||||
timeline_type = 'media'
|
||||
|
||||
account_info = data.get('account_info', {})
|
||||
timeline = data.get('timeline', [])
|
||||
|
||||
username = data.get('username') or account_info.get('name', filename_parts[0] if filename_parts else 'unknown')
|
||||
|
||||
nick = data.get('nick') or account_info.get('nick') or account_info.get('name', username)
|
||||
followers = data.get('followers') or account_info.get('followers_count', 0)
|
||||
following = data.get('following') or account_info.get('friends_count', 0)
|
||||
posts = data.get('posts') or account_info.get('statuses_count', 0)
|
||||
profile_image = data.get('profile_image') or account_info.get('profile_image')
|
||||
|
||||
self.save_account(
|
||||
username=username,
|
||||
nick=nick,
|
||||
followers=followers,
|
||||
following=following,
|
||||
posts=posts,
|
||||
media_type=media_type,
|
||||
profile_image=profile_image,
|
||||
fetch_mode=data.get('fetch_mode', 'all'),
|
||||
timeline_type=timeline_type
|
||||
)
|
||||
accounts_migrated += 1
|
||||
|
||||
media_list = timeline if timeline else data.get('media_list', [])
|
||||
if media_list:
|
||||
self.save_media_list(username, media_list)
|
||||
media_migrated += len(media_list)
|
||||
|
||||
backup_path = json_path + '.backup'
|
||||
if not os.path.exists(backup_path):
|
||||
os.rename(json_path, backup_path)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error migrating {json_file}: {e}")
|
||||
continue
|
||||
|
||||
return accounts_migrated, media_migrated
|
||||
|
||||
def export_to_json(self, username: str, media_type: str, output_path: str):
|
||||
account = self.get_account(username, media_type)
|
||||
if not account:
|
||||
return False
|
||||
|
||||
media_list = self.get_media_list(username)
|
||||
|
||||
export_data = {
|
||||
'username': account['username'],
|
||||
'nick': account['nick'],
|
||||
'followers': account['followers'],
|
||||
'following': account['following'],
|
||||
'posts': account['posts'],
|
||||
'profile_image': account['profile_image'],
|
||||
'fetch_timestamp': account['fetch_timestamp'],
|
||||
'media_type': account['media_type'],
|
||||
'fetch_mode': account['fetch_mode'],
|
||||
'media_list': media_list
|
||||
}
|
||||
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(export_data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
return True
|
||||
|
||||
def import_from_json(self, json_path: str) -> bool:
|
||||
try:
|
||||
with open(json_path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
username = data.get('username')
|
||||
if not username:
|
||||
return False
|
||||
|
||||
self.save_account(
|
||||
username=username,
|
||||
nick=data.get('nick', ''),
|
||||
followers=data.get('followers', 0),
|
||||
following=data.get('following', 0),
|
||||
posts=data.get('posts', 0),
|
||||
media_type=data.get('media_type', 'all'),
|
||||
profile_image=data.get('profile_image'),
|
||||
fetch_mode=data.get('fetch_mode', 'all')
|
||||
)
|
||||
|
||||
media_list = data.get('media_list', [])
|
||||
if media_list:
|
||||
self.save_media_list(username, media_list)
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error importing JSON: {e}")
|
||||
return False
|
||||
|
||||
def close(self):
|
||||
if self.conn:
|
||||
self.conn.close()
|
||||
|
||||
def create_group(self, name: str) -> Optional[int]:
|
||||
with self._lock:
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
created_at = datetime.now().isoformat()
|
||||
cursor.execute('INSERT INTO groups (name, created_at) VALUES (?, ?)', (name, created_at))
|
||||
self.conn.commit()
|
||||
return cursor.lastrowid
|
||||
except sqlite3.IntegrityError:
|
||||
return None
|
||||
|
||||
def get_all_groups(self) -> List[Dict]:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute('SELECT * FROM groups ORDER BY name')
|
||||
return [dict(row) for row in cursor.fetchall()]
|
||||
|
||||
def get_group(self, group_id: int) -> Optional[Dict]:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute('SELECT * FROM groups WHERE id = ?', (group_id,))
|
||||
row = cursor.fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def update_group(self, group_id: int, name: str) -> bool:
|
||||
with self._lock:
|
||||
try:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute('UPDATE groups SET name = ? WHERE id = ?', (name, group_id))
|
||||
self.conn.commit()
|
||||
return True
|
||||
except sqlite3.IntegrityError:
|
||||
return False
|
||||
|
||||
def delete_group(self, group_id: int):
|
||||
with self._lock:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute('UPDATE accounts SET group_id = NULL WHERE group_id = ?', (group_id,))
|
||||
cursor.execute('DELETE FROM groups WHERE id = ?', (group_id,))
|
||||
self.conn.commit()
|
||||
|
||||
def assign_account_to_group(self, username: str, group_id: Optional[int]):
|
||||
with self._lock:
|
||||
cursor = self.conn.cursor()
|
||||
cursor.execute('UPDATE accounts SET group_id = ? WHERE username = ?', (group_id, username))
|
||||
self.conn.commit()
|
||||
|
||||
def get_accounts_by_group(self, group_id: Optional[int]) -> List[Dict]:
|
||||
cursor = self.conn.cursor()
|
||||
if group_id is None:
|
||||
cursor.execute('SELECT * FROM accounts WHERE group_id IS NULL ORDER BY fetch_timestamp DESC')
|
||||
else:
|
||||
cursor.execute('SELECT * FROM accounts WHERE group_id = ? ORDER BY fetch_timestamp DESC', (group_id,))
|
||||
return [dict(row) for row in cursor.fetchall()]
|
||||
@@ -0,0 +1,307 @@
|
||||
import sys
|
||||
import os
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import subprocess
|
||||
import imageio_ffmpeg
|
||||
from datetime import datetime
|
||||
from PyQt6.QtCore import QThread, pyqtSignal
|
||||
|
||||
DEFAULT_BATCH_SIZE = 25
|
||||
TIMEOUT_SECONDS_MAC = 60
|
||||
TIMEOUT_SECONDS_DEFAULT = 30
|
||||
BATCH_SLEEP_INTERVAL = 0.1
|
||||
PAUSE_CHECK_INTERVAL = 0.1
|
||||
|
||||
class DownloadWorker(QThread):
|
||||
finished = pyqtSignal(bool, str)
|
||||
progress = pyqtSignal(str, int)
|
||||
conversion_progress = pyqtSignal(str, int)
|
||||
download_progress = pyqtSignal(str, int)
|
||||
|
||||
def __init__(self, accounts, outpath, auth_token, filename_format='username_date',
|
||||
download_batch_size=DEFAULT_BATCH_SIZE, convert_gif=False, gif_resolution='original', gif_conversion_mode='better', db_manager=None):
|
||||
super().__init__()
|
||||
self.accounts = accounts
|
||||
self.outpath = outpath
|
||||
self.auth_token = auth_token
|
||||
self.filename_format = filename_format
|
||||
self.download_batch_size = download_batch_size
|
||||
self.convert_gif = convert_gif
|
||||
self.gif_resolution = gif_resolution
|
||||
self.gif_conversion_mode = gif_conversion_mode
|
||||
self.is_paused = False
|
||||
self.is_stopped = False
|
||||
self.filepath_map = []
|
||||
self.db_manager = db_manager
|
||||
|
||||
async def download_file(self, session, url, filepath, username, tweet_id):
|
||||
try:
|
||||
if self.db_manager and self.db_manager.is_url_downloaded(url):
|
||||
return True, True
|
||||
|
||||
if os.path.exists(filepath):
|
||||
if self.db_manager:
|
||||
file_size = os.path.getsize(filepath)
|
||||
self.db_manager.record_download(url, filepath, username, tweet_id, 'downloaded', file_size)
|
||||
return True, True
|
||||
|
||||
if self.is_stopped:
|
||||
return False, False
|
||||
|
||||
try:
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
except OSError as e:
|
||||
return False, False
|
||||
|
||||
async with session.get(url) as response:
|
||||
if response.status == 200:
|
||||
content = await response.read()
|
||||
|
||||
try:
|
||||
with open(filepath, 'wb') as f:
|
||||
f.write(content)
|
||||
except (IOError, OSError) as e:
|
||||
if self.db_manager:
|
||||
self.db_manager.record_download(url, filepath, username, tweet_id, 'failed', 0)
|
||||
return False, False
|
||||
|
||||
if self.db_manager:
|
||||
self.db_manager.record_download(url, filepath, username, tweet_id, 'downloaded', len(content))
|
||||
|
||||
return True, False
|
||||
return False, False
|
||||
except asyncio.TimeoutError:
|
||||
if self.db_manager:
|
||||
self.db_manager.record_download(url, filepath, username, tweet_id, 'failed', 0)
|
||||
return False, False
|
||||
except aiohttp.ClientError as e:
|
||||
if self.db_manager:
|
||||
self.db_manager.record_download(url, filepath, username, tweet_id, 'failed', 0)
|
||||
return False, False
|
||||
except Exception as e:
|
||||
if self.db_manager:
|
||||
self.db_manager.record_download(url, filepath, username, tweet_id, 'failed', 0)
|
||||
return False, False
|
||||
|
||||
async def download_account_media(self, account):
|
||||
if not account.media_list:
|
||||
return 0, 0, 0
|
||||
|
||||
safe_username = os.path.basename(account.username.replace('/', '_').replace('\\', '_'))
|
||||
account_output_dir = os.path.join(self.outpath, safe_username)
|
||||
os.makedirs(account_output_dir, exist_ok=True)
|
||||
|
||||
timeout_seconds = TIMEOUT_SECONDS_MAC if sys.platform == 'darwin' else TIMEOUT_SECONDS_DEFAULT
|
||||
timeout = aiohttp.ClientTimeout(total=timeout_seconds)
|
||||
connector = aiohttp.TCPConnector(limit=self.download_batch_size)
|
||||
|
||||
async with aiohttp.ClientSession(timeout=timeout, connector=connector) as session:
|
||||
total = len(account.media_list)
|
||||
completed = 0
|
||||
skipped = 0
|
||||
failed = 0
|
||||
|
||||
used_filenames = set()
|
||||
|
||||
for i in range(0, total, self.download_batch_size):
|
||||
if self.is_stopped:
|
||||
break
|
||||
|
||||
while self.is_paused:
|
||||
if self.is_stopped:
|
||||
return completed, skipped, failed
|
||||
await asyncio.sleep(PAUSE_CHECK_INTERVAL)
|
||||
|
||||
batch = account.media_list[i:i + self.download_batch_size]
|
||||
tasks = []
|
||||
|
||||
for item in batch:
|
||||
url = item['url']
|
||||
date = datetime.strptime(item['date'], "%Y-%m-%d %H:%M:%S")
|
||||
formatted_date = date.strftime("%Y%m%d_%H%M%S")
|
||||
tweet_id = str(item.get('tweet_id', ''))
|
||||
|
||||
item_type = item.get('type', '')
|
||||
if item_type == 'animated_gif':
|
||||
media_type_folder = 'gif'
|
||||
extension = 'mp4'
|
||||
elif item_type == 'video' or 'video.twimg.com' in url:
|
||||
media_type_folder = 'video'
|
||||
extension = 'mp4'
|
||||
else:
|
||||
media_type_folder = 'image'
|
||||
extension = 'jpg'
|
||||
media_output_dir = os.path.join(account_output_dir, media_type_folder)
|
||||
|
||||
if self.filename_format == "username_date":
|
||||
base_filename = f"{safe_username}_{formatted_date}_{tweet_id}"
|
||||
else:
|
||||
base_filename = f"{formatted_date}_{safe_username}_{tweet_id}"
|
||||
|
||||
filename = f"{base_filename}.{extension}"
|
||||
counter = 1
|
||||
while filename in used_filenames:
|
||||
filename = f"{base_filename}_{counter:02d}.{extension}"
|
||||
counter += 1
|
||||
|
||||
used_filenames.add(filename)
|
||||
filepath = os.path.join(media_output_dir, filename)
|
||||
self.filepath_map.append((item, filepath))
|
||||
task = asyncio.create_task(self.download_file(session, url, filepath, account.username, tweet_id))
|
||||
tasks.append(task)
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
for result in results:
|
||||
if isinstance(result, tuple):
|
||||
success, was_skipped = result
|
||||
if success:
|
||||
if was_skipped:
|
||||
skipped += 1
|
||||
else:
|
||||
completed += 1
|
||||
else:
|
||||
failed += 1
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
progress_percent = int((completed + skipped + failed) / total * 100)
|
||||
media_type_display = account.media_type if account.media_type != 'all' else 'media'
|
||||
self.download_progress.emit(f"Downloading {account.username}'s {media_type_display}: {completed + skipped + failed:,}/{total:,}", progress_percent)
|
||||
|
||||
await asyncio.sleep(BATCH_SLEEP_INTERVAL)
|
||||
|
||||
return completed, skipped, failed
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
if sys.platform == 'darwin':
|
||||
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
|
||||
|
||||
total_accounts = len(self.accounts)
|
||||
overall_completed = 0
|
||||
overall_skipped = 0
|
||||
overall_failed = 0
|
||||
|
||||
for i, account in enumerate(self.accounts):
|
||||
if self.is_stopped:
|
||||
break
|
||||
|
||||
while self.is_paused:
|
||||
if self.is_stopped:
|
||||
return
|
||||
self.msleep(100)
|
||||
media_type_display = account.media_type if account.media_type != 'all' else 'media'
|
||||
self.progress.emit(f"Downloading from account: {account.username} ({media_type_display}) - ({i+1}/{total_accounts})",
|
||||
int((i) / total_accounts * 100))
|
||||
|
||||
completed, skipped, failed = asyncio.run(self.download_account_media(account))
|
||||
overall_completed += completed
|
||||
overall_skipped += skipped
|
||||
overall_failed += failed
|
||||
self.progress.emit(f"Account {account.username} ({media_type_display}): {completed:,} downloaded, {skipped:,} skipped, {failed:,} failed",
|
||||
int((i + 1) / total_accounts * 100))
|
||||
|
||||
if not self.is_stopped:
|
||||
if self.convert_gif:
|
||||
try:
|
||||
ffmpeg_exe = imageio_ffmpeg.get_ffmpeg_exe()
|
||||
gif_items = [(item, fp) for item, fp in self.filepath_map if item.get('type') == 'animated_gif']
|
||||
total_gifs = len(gif_items)
|
||||
if total_gifs > 0:
|
||||
if sys.platform == 'win32':
|
||||
creationflags = subprocess.CREATE_NO_WINDOW
|
||||
else:
|
||||
creationflags = 0
|
||||
|
||||
converted_count = 0
|
||||
skipped_count = 0
|
||||
self.progress.emit("Starting GIF conversion...", 0)
|
||||
|
||||
for idx, (item, fp) in enumerate(gif_items, start=1):
|
||||
if self.is_stopped:
|
||||
break
|
||||
|
||||
while self.is_paused:
|
||||
if self.is_stopped:
|
||||
break
|
||||
self.msleep(100)
|
||||
|
||||
if self.is_stopped:
|
||||
break
|
||||
|
||||
gif_fp = fp.rsplit('.', 1)[0] + '.gif'
|
||||
|
||||
if os.path.exists(gif_fp):
|
||||
try:
|
||||
os.remove(fp)
|
||||
skipped_count += 1
|
||||
conv_progress = int((idx / total_gifs) * 100)
|
||||
quality_display = f"({self.gif_conversion_mode}) - ({self.gif_resolution})"
|
||||
self.conversion_progress.emit(f"Converting GIF {idx:,}/{total_gifs:,} {quality_display} - Skipped (exists)", conv_progress)
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
if self.gif_conversion_mode == 'fast':
|
||||
ffmpeg_args = [ffmpeg_exe, '-i', fp, '-y', gif_fp]
|
||||
else:
|
||||
if self.gif_resolution == 'high':
|
||||
ffmpeg_args = [ffmpeg_exe, '-i', fp, '-lavfi', 'scale=800:-1:flags=lanczos,palettegen=stats_mode=full[palette];[0:v]scale=800:-1:flags=lanczos[scaled];[scaled][palette]paletteuse=dither=sierra2_4a', '-r', '15', '-y', gif_fp]
|
||||
elif self.gif_resolution == 'medium':
|
||||
ffmpeg_args = [ffmpeg_exe, '-i', fp, '-lavfi', 'scale=600:-1:flags=lanczos,palettegen=stats_mode=full[palette];[0:v]scale=600:-1:flags=lanczos[scaled];[scaled][palette]paletteuse=dither=sierra2_4a', '-r', '10', '-y', gif_fp]
|
||||
elif self.gif_resolution == 'low':
|
||||
ffmpeg_args = [ffmpeg_exe, '-i', fp, '-lavfi', 'scale=400:-1:flags=lanczos,palettegen=stats_mode=full[palette];[0:v]scale=400:-1:flags=lanczos[scaled];[scaled][palette]paletteuse=dither=sierra2_4a', '-r', '8', '-y', gif_fp]
|
||||
else:
|
||||
ffmpeg_args = [ffmpeg_exe, '-i', fp, '-lavfi', 'palettegen=stats_mode=full[palette];[0:v][palette]paletteuse=dither=sierra2_4a', '-y', gif_fp]
|
||||
|
||||
try:
|
||||
if sys.platform == 'win32':
|
||||
result = subprocess.run(ffmpeg_args, capture_output=True, creationflags=creationflags)
|
||||
else:
|
||||
result = subprocess.run(ffmpeg_args, capture_output=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
|
||||
|
||||
if result.returncode == 0 and os.path.exists(gif_fp):
|
||||
try:
|
||||
os.remove(fp)
|
||||
converted_count += 1
|
||||
except Exception:
|
||||
pass
|
||||
elif result.returncode != 0:
|
||||
error_msg = result.stderr.decode('utf-8', errors='ignore') if result.stderr else 'Unknown error'
|
||||
self.progress.emit(f"GIF conversion warning for file {idx}: {error_msg[:100]}", 0)
|
||||
except Exception as e:
|
||||
self.progress.emit(f"GIF conversion error for file {idx}: {str(e)[:100]}", 0)
|
||||
|
||||
conv_progress = int((idx / total_gifs) * 100)
|
||||
quality_display = f"({self.gif_conversion_mode}) - ({self.gif_resolution})"
|
||||
self.conversion_progress.emit(f"Converting GIF {idx:,}/{total_gifs:,} {quality_display}", conv_progress)
|
||||
if converted_count > 0 or skipped_count > 0:
|
||||
completion_msg = f"GIF conversion completed: {converted_count:,} converted"
|
||||
if skipped_count > 0:
|
||||
completion_msg += f", {skipped_count:,} skipped (already exists)"
|
||||
self.progress.emit(completion_msg, 100)
|
||||
else:
|
||||
self.progress.emit("GIF conversion completed", 100)
|
||||
except Exception as conv_e:
|
||||
self.progress.emit(f"GIF conversion error: {conv_e}", 0)
|
||||
|
||||
success_message = f"Download completed! {overall_completed:,} files downloaded"
|
||||
if overall_skipped > 0:
|
||||
success_message += f", {overall_skipped:,} skipped"
|
||||
if overall_failed > 0:
|
||||
success_message += f", {overall_failed:,} failed"
|
||||
self.finished.emit(True, success_message)
|
||||
|
||||
except Exception as e:
|
||||
self.finished.emit(False, str(e))
|
||||
|
||||
def pause(self):
|
||||
self.is_paused = True
|
||||
|
||||
def resume(self):
|
||||
self.is_paused = False
|
||||
|
||||
def stop(self):
|
||||
self.is_stopped = True
|
||||
self.is_paused = False
|
||||
@@ -3,7 +3,143 @@ import json
|
||||
from datetime import datetime
|
||||
from gallery_dl.extractor import twitter
|
||||
|
||||
def get_metadata(username, auth_token, timeline_type="timeline", batch_size=0, page=0, media_type="all"):
|
||||
def get_metadata_by_date(username, auth_token, date_start, date_end, media_filter="filter:media", output_file=None):
|
||||
query = f"from:{username} since:{date_start} until:{date_end}"
|
||||
if media_filter:
|
||||
query += f" {media_filter}"
|
||||
|
||||
url = f"https://x.com/search?q={query}"
|
||||
|
||||
extractor_class = twitter.TwitterSearchExtractor
|
||||
match = re.match(extractor_class.pattern, url)
|
||||
|
||||
if not match:
|
||||
raise ValueError(f"Invalid search URL: {url}")
|
||||
|
||||
extractor = extractor_class(match)
|
||||
|
||||
config_dict = {
|
||||
"cookies": {
|
||||
"auth_token": auth_token
|
||||
},
|
||||
"retweets": False
|
||||
}
|
||||
|
||||
extractor.config = lambda key, default=None: config_dict.get(key, default)
|
||||
|
||||
try:
|
||||
extractor.initialize()
|
||||
|
||||
api = twitter.TwitterAPI(extractor)
|
||||
|
||||
try:
|
||||
user = api.user_by_screen_name(username)
|
||||
|
||||
if "legacy" in user and user["legacy"].get("withheld_scope"):
|
||||
raise ValueError("withheld")
|
||||
|
||||
except Exception as e:
|
||||
error_msg = str(e).lower()
|
||||
if "withheld" in error_msg or (hasattr(e, "response") and "withheld" in str(e.response.text).lower()):
|
||||
raise ValueError("withheld")
|
||||
raise
|
||||
|
||||
user_data = extractor._transform_user(user)
|
||||
|
||||
structured_output = {
|
||||
'account_info': {
|
||||
'name': user_data.get('name', ''),
|
||||
'nick': user_data.get('nick', ''),
|
||||
'date': user_data.get('date', ''),
|
||||
'followers_count': user_data.get('followers_count', 0),
|
||||
'friends_count': user_data.get('friends_count', 0),
|
||||
'profile_image': user_data.get('profile_image', ''),
|
||||
'statuses_count': user_data.get('statuses_count', 0)
|
||||
},
|
||||
'total_urls': 0,
|
||||
'timeline': [],
|
||||
'search_query': query,
|
||||
'date_filter': {
|
||||
'start': date_start,
|
||||
'end': date_end,
|
||||
'method': 'search_api'
|
||||
}
|
||||
}
|
||||
|
||||
if isinstance(structured_output['account_info']['date'], datetime):
|
||||
structured_output['account_info']['date'] = structured_output['account_info']['date'].strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
new_timeline_entries = []
|
||||
|
||||
try:
|
||||
iterator = iter(extractor)
|
||||
|
||||
while True:
|
||||
try:
|
||||
item = next(iterator)
|
||||
|
||||
if isinstance(item, tuple) and len(item) >= 3:
|
||||
media_url = item[1]
|
||||
tweet_data = item[2]
|
||||
|
||||
if 'pbs.twimg.com' in media_url or 'video.twimg.com' in media_url:
|
||||
tweet_date = tweet_data.get('date', datetime.now())
|
||||
if isinstance(tweet_date, datetime):
|
||||
tweet_date = tweet_date.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
timeline_entry = {
|
||||
'url': media_url,
|
||||
'date': tweet_date,
|
||||
'tweet_id': tweet_data.get('tweet_id', 0),
|
||||
}
|
||||
|
||||
if 'type' in tweet_data:
|
||||
timeline_entry['type'] = tweet_data['type']
|
||||
|
||||
if 'retweet_id' in tweet_data and tweet_data['retweet_id']:
|
||||
timeline_entry['retweet_id'] = tweet_data['retweet_id']
|
||||
timeline_entry['is_retweet'] = True
|
||||
else:
|
||||
timeline_entry['is_retweet'] = False
|
||||
|
||||
new_timeline_entries.append(timeline_entry)
|
||||
structured_output['total_urls'] += 1
|
||||
|
||||
except StopIteration:
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
structured_output['timeline'] = new_timeline_entries
|
||||
|
||||
structured_output['metadata'] = {
|
||||
"new_entries": len(new_timeline_entries),
|
||||
"method": "search_api",
|
||||
"date_range": f"{date_start} to {date_end}"
|
||||
}
|
||||
|
||||
if output_file:
|
||||
try:
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(structured_output, f, ensure_ascii=False, indent=2)
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
return structured_output
|
||||
|
||||
except Exception as e:
|
||||
error_msg = str(e).lower()
|
||||
if "withheld" in error_msg or e.__class__.__name__ == "ValueError" and str(e) == "withheld":
|
||||
return {"error": "To download withheld accounts, use this userscript version: https://www.patreon.com/exyezed"}
|
||||
else:
|
||||
error_str = str(e)
|
||||
if error_str == "None":
|
||||
return {"error": "Failed to authenticate. Please verify your auth token is valid and not expired."}
|
||||
else:
|
||||
return {"error": error_str}
|
||||
|
||||
def get_metadata(username, auth_token, timeline_type="timeline", batch_size=0, page=0, media_type="all", retweets=False):
|
||||
url = f"https://x.com/{username}/{timeline_type}"
|
||||
|
||||
if timeline_type == "media":
|
||||
@@ -24,7 +160,8 @@ def get_metadata(username, auth_token, timeline_type="timeline", batch_size=0, p
|
||||
config_dict = {
|
||||
"cookies": {
|
||||
"auth_token": auth_token
|
||||
}
|
||||
},
|
||||
"retweets": retweets
|
||||
}
|
||||
|
||||
if batch_size > 0:
|
||||
@@ -117,6 +254,12 @@ def get_metadata(username, auth_token, timeline_type="timeline", batch_size=0, p
|
||||
if 'type' in tweet_data:
|
||||
timeline_entry['type'] = tweet_data['type']
|
||||
|
||||
if 'retweet_id' in tweet_data and tweet_data['retweet_id']:
|
||||
timeline_entry['retweet_id'] = tweet_data['retweet_id']
|
||||
timeline_entry['is_retweet'] = True
|
||||
else:
|
||||
timeline_entry['is_retweet'] = False
|
||||
|
||||
if media_type == 'all' or (
|
||||
(media_type == 'image' and 'pbs.twimg.com' in media_url and tweet_data.get('type') == 'photo') or
|
||||
(media_type == 'video' and 'video.twimg.com' in media_url and tweet_data.get('type') == 'video') or
|
||||
@@ -0,0 +1,60 @@
|
||||
from PyQt6.QtCore import QThread, pyqtSignal
|
||||
from core.metadataFetcher import get_metadata, get_metadata_by_date
|
||||
|
||||
class MetadataFetchWorker(QThread):
|
||||
finished = pyqtSignal(dict)
|
||||
error = pyqtSignal(str)
|
||||
|
||||
def __init__(self, username, media_type='all', batch_mode=False, batch_size=0, page=0, timeline_type='media', include_retweets=False, use_date_range=False, date_start=None, date_end=None):
|
||||
super().__init__()
|
||||
self.username = username
|
||||
self.media_type = media_type
|
||||
self.batch_mode = batch_mode
|
||||
self.batch_size = batch_size
|
||||
self.page = page
|
||||
self.timeline_type = timeline_type
|
||||
self.include_retweets = include_retweets
|
||||
self.use_date_range = use_date_range
|
||||
self.date_start = date_start
|
||||
self.date_end = date_end
|
||||
self.auth_token = None
|
||||
|
||||
def normalize_url(self, url_or_username):
|
||||
url_or_username = url_or_username.strip()
|
||||
username = url_or_username
|
||||
|
||||
if "x.com/" in url_or_username or "twitter.com/" in url_or_username:
|
||||
parts = url_or_username.split('/')
|
||||
for i, part in enumerate(parts):
|
||||
if part in ['x.com', 'twitter.com'] and i + 1 < len(parts):
|
||||
username = parts[i + 1]
|
||||
username = username.split('/')[0]
|
||||
break
|
||||
|
||||
username = username.strip()
|
||||
return username
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
normalized = self.normalize_url(self.username)
|
||||
if self.use_date_range and self.date_start and self.date_end:
|
||||
data = get_metadata_by_date(
|
||||
username=normalized,
|
||||
auth_token=self.auth_token,
|
||||
date_start=self.date_start,
|
||||
date_end=self.date_end,
|
||||
media_filter="filter:media" if self.media_type == 'all' else f"filter:{self.media_type}"
|
||||
)
|
||||
else:
|
||||
data = get_metadata(
|
||||
username=normalized,
|
||||
auth_token=self.auth_token,
|
||||
timeline_type=self.timeline_type,
|
||||
batch_size=self.batch_size if self.batch_mode else 0,
|
||||
page=self.page,
|
||||
media_type=self.media_type,
|
||||
retweets=self.include_retweets
|
||||
)
|
||||
self.finished.emit(data)
|
||||
except Exception as e:
|
||||
self.error.emit(str(e))
|
||||
@@ -1,6 +1,8 @@
|
||||
PyQt6
|
||||
PyQt6==6.9.1
|
||||
requests
|
||||
aiohttp
|
||||
imageio-ffmpeg
|
||||
packaging
|
||||
gallery-dl
|
||||
gallery-dl
|
||||
pyqtdarktheme
|
||||
pyinstaller
|
||||
@@ -0,0 +1,7 @@
|
||||
from ui.mainWindow import TwitterXMediaBatchDownloaderGUI
|
||||
from ui.dialogs import UpdateDialog
|
||||
|
||||
__all__ = [
|
||||
'TwitterXMediaBatchDownloaderGUI',
|
||||
'UpdateDialog'
|
||||
]
|
||||
@@ -0,0 +1,33 @@
|
||||
from PyQt6.QtWidgets import (
|
||||
QDialog, QVBoxLayout, QLabel, QPushButton, QDialogButtonBox
|
||||
)
|
||||
from PyQt6.QtCore import Qt
|
||||
|
||||
class UpdateDialog(QDialog):
|
||||
def __init__(self, current_version, new_version, parent=None):
|
||||
super().__init__(parent)
|
||||
self.setWindowTitle("Update Now")
|
||||
self.setFixedWidth(400)
|
||||
self.setModal(True)
|
||||
|
||||
layout = QVBoxLayout()
|
||||
|
||||
message = QLabel(f"Twitter/X Media Batch Downloader v{new_version} Available!")
|
||||
message.setWordWrap(True)
|
||||
layout.addWidget(message)
|
||||
|
||||
button_box = QDialogButtonBox()
|
||||
self.update_button = QPushButton("Check")
|
||||
self.update_button.setCursor(Qt.CursorShape.PointingHandCursor)
|
||||
self.cancel_button = QPushButton("Later")
|
||||
self.cancel_button.setCursor(Qt.CursorShape.PointingHandCursor)
|
||||
|
||||
button_box.addButton(self.update_button, QDialogButtonBox.ButtonRole.AcceptRole)
|
||||
button_box.addButton(self.cancel_button, QDialogButtonBox.ButtonRole.RejectRole)
|
||||
|
||||
layout.addWidget(button_box)
|
||||
|
||||
self.setLayout(layout)
|
||||
|
||||
self.update_button.clicked.connect(self.accept)
|
||||
self.cancel_button.clicked.connect(self.reject)
|
||||
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round" class="icon icon-tabler icons-tabler-outline icon-tabler-cancel"><path stroke="none" d="M0 0h24v24H0z" fill="none"/><path d="M3 12a9 9 0 1 0 18 0a9 9 0 1 0 -18 0" /><path d="M18.364 5.636l-12.728 12.728" /></svg>
|
||||
|
After Width: | Height: | Size: 396 B |
|
Before Width: | Height: | Size: 616 B After Width: | Height: | Size: 616 B |
|
Before Width: | Height: | Size: 610 B After Width: | Height: | Size: 610 B |
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round" class="icon icon-tabler icons-tabler-outline icon-tabler-device-floppy"><path stroke="none" d="M0 0h24v24H0z" fill="none"/><path d="M6 4h10l4 4v10a2 2 0 0 1 -2 2h-12a2 2 0 0 1 -2 -2v-12a2 2 0 0 1 2 -2" /><path d="M12 14m-2 0a2 2 0 1 0 4 0a2 2 0 1 0 -4 0" /><path d="M14 4l0 4l-6 0l0 -4" /></svg>
|
||||
|
After Width: | Height: | Size: 479 B |
|
Before Width: | Height: | Size: 423 B After Width: | Height: | Size: 423 B |
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round" class="icon icon-tabler icons-tabler-outline icon-tabler-edit"><path stroke="none" d="M0 0h24v24H0z" fill="none"/><path d="M7 7h-1a2 2 0 0 0 -2 2v9a2 2 0 0 0 2 2h9a2 2 0 0 0 2 -2v-1" /><path d="M20.385 6.585a2.1 2.1 0 0 0 -2.97 -2.97l-8.415 8.385v3h3l8.385 -8.415z" /><path d="M16 5l3 3" /></svg>
|
||||
|
After Width: | Height: | Size: 480 B |
|
Before Width: | Height: | Size: 173 KiB After Width: | Height: | Size: 173 KiB |
|
Before Width: | Height: | Size: 865 B After Width: | Height: | Size: 865 B |
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round" class="icon icon-tabler icons-tabler-outline icon-tabler-player-pause"><path stroke="none" d="M0 0h24v24H0z" fill="none"/><path d="M6 5m0 1a1 1 0 0 1 1 -1h2a1 1 0 0 1 1 1v12a1 1 0 0 1 -1 1h-2a1 1 0 0 1 -1 -1z" /><path d="M14 5m0 1a1 1 0 0 1 1 -1h2a1 1 0 0 1 1 1v12a1 1 0 0 1 -1 1h-2a1 1 0 0 1 -1 -1z" /></svg>
|
||||
|
After Width: | Height: | Size: 493 B |
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round" class="icon icon-tabler icons-tabler-outline icon-tabler-player-play"><path stroke="none" d="M0 0h24v24H0z" fill="none"/><path d="M7 4v16l13 -8z" /></svg>
|
||||
|
After Width: | Height: | Size: 338 B |
|
Before Width: | Height: | Size: 423 B After Width: | Height: | Size: 423 B |
|
Before Width: | Height: | Size: 495 B After Width: | Height: | Size: 495 B |
@@ -0,0 +1,267 @@
|
||||
import os
|
||||
from PyQt6.QtWidgets import QApplication, QLabel
|
||||
from PyQt6.QtCore import Qt, QSize, QEvent
|
||||
from PyQt6.QtGui import QIcon, QPalette, QColor
|
||||
from ui.mainWindow import TwitterXMediaBatchDownloaderGUI
|
||||
|
||||
class TwitterXMediaBatchDownloaderGUI_Unix(TwitterXMediaBatchDownloaderGUI):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.setup_unix_specific()
|
||||
|
||||
QApplication.instance().installEventFilter(self)
|
||||
|
||||
def eventFilter(self, obj, event):
|
||||
if event.type() == QEvent.Type.ApplicationPaletteChange:
|
||||
self.update_button_icons_unix()
|
||||
self.update_ui_colors()
|
||||
return super().eventFilter(obj, event)
|
||||
|
||||
def update_ui_colors(self):
|
||||
from PyQt6.QtWidgets import QLineEdit, QTextEdit, QComboBox, QCheckBox, QPushButton, QProgressBar, QDateEdit, QListWidget, QTabWidget
|
||||
|
||||
new_palette = QApplication.palette()
|
||||
|
||||
self.account_list.setStyleSheet("""
|
||||
QListWidget {
|
||||
background-color: palette(base);
|
||||
padding: 4px;
|
||||
outline: none;
|
||||
}
|
||||
QListWidget::item {
|
||||
padding: 3px 6px;
|
||||
margin: 0px 0px;
|
||||
border: none;
|
||||
outline: none;
|
||||
}
|
||||
QListWidget::item:selected {
|
||||
background-color: palette(highlight);
|
||||
border: none;
|
||||
outline: none;
|
||||
}
|
||||
QListWidget::item:focus {
|
||||
border: none;
|
||||
outline: none;
|
||||
}
|
||||
QListWidget::item:hover {
|
||||
background-color: palette(highlight);
|
||||
}
|
||||
""")
|
||||
|
||||
widget_types = [QLabel, QLineEdit, QTextEdit, QComboBox, QCheckBox, QPushButton, QProgressBar, QDateEdit, QListWidget, QTabWidget]
|
||||
|
||||
for widget_type in widget_types:
|
||||
for widget in self.findChildren(widget_type):
|
||||
widget.setPalette(new_palette)
|
||||
widget.update()
|
||||
|
||||
for i in range(self.account_list.count()):
|
||||
item = self.account_list.item(i)
|
||||
if item:
|
||||
self.account_list.update(self.account_list.indexFromItem(item))
|
||||
|
||||
self.setPalette(new_palette)
|
||||
self.update()
|
||||
|
||||
def setup_unix_specific(self):
|
||||
self.setFixedWidth(670)
|
||||
|
||||
if hasattr(self, 'fetch_btn'):
|
||||
self.fetch_btn.setFixedWidth(100)
|
||||
|
||||
if hasattr(self, 'start_date_edit'):
|
||||
self.start_date_edit.setMinimumWidth(130)
|
||||
|
||||
if hasattr(self, 'end_date_edit'):
|
||||
self.end_date_edit.setMinimumWidth(130)
|
||||
|
||||
if hasattr(self, 'output_browse'):
|
||||
self.output_browse.setFixedWidth(100)
|
||||
|
||||
self.account_list.setStyleSheet("""
|
||||
QListWidget {
|
||||
background-color: palette(base);
|
||||
padding: 4px;
|
||||
outline: none;
|
||||
}
|
||||
QListWidget::item {
|
||||
padding: 3px 6px;
|
||||
margin: 0px 0px;
|
||||
border: none;
|
||||
outline: none;
|
||||
}
|
||||
QListWidget::item:selected {
|
||||
background-color: palette(highlight);
|
||||
border: none;
|
||||
outline: none;
|
||||
}
|
||||
QListWidget::item:focus {
|
||||
border: none;
|
||||
outline: none;
|
||||
}
|
||||
QListWidget::item:hover {
|
||||
background-color: palette(highlight);
|
||||
}
|
||||
""")
|
||||
|
||||
self.update_button_icons_unix()
|
||||
|
||||
def create_colored_icon(self, icon_path, color=None):
|
||||
if not os.path.exists(icon_path):
|
||||
return QIcon()
|
||||
|
||||
palette = QApplication.palette()
|
||||
text_color = palette.color(QPalette.ColorRole.Text)
|
||||
color_hex = text_color.name()
|
||||
|
||||
from PyQt6.QtSvg import QSvgRenderer
|
||||
from PyQt6.QtGui import QPixmap, QPainter
|
||||
|
||||
renderer = QSvgRenderer(icon_path)
|
||||
pixmap = QPixmap(24, 24)
|
||||
pixmap.fill(Qt.GlobalColor.transparent)
|
||||
|
||||
painter = QPainter(pixmap)
|
||||
renderer.render(painter)
|
||||
painter.setCompositionMode(QPainter.CompositionMode.CompositionMode_SourceIn)
|
||||
painter.fillRect(pixmap.rect(), QColor(color_hex))
|
||||
painter.end()
|
||||
|
||||
return QIcon(pixmap)
|
||||
|
||||
def update_button_icons(self):
|
||||
self.update_button_icons_unix()
|
||||
|
||||
def update_button_icons_unix_dashboard(self):
|
||||
if not hasattr(self, 'import_btn'):
|
||||
return
|
||||
|
||||
icon_dir = os.path.join(os.path.dirname(__file__), 'icons')
|
||||
|
||||
self.import_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'database-import.svg')))
|
||||
self.export_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'database-export.svg')))
|
||||
self.download_selected_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'download.svg')))
|
||||
self.update_selected_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'reload.svg')))
|
||||
self.delete_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'trash.svg')))
|
||||
|
||||
def update_button_icons_unix(self):
|
||||
self.update_button_icons_unix_dashboard()
|
||||
|
||||
icon_dir = os.path.join(os.path.dirname(__file__), 'icons')
|
||||
|
||||
if hasattr(self, 'stop_btn'):
|
||||
self.stop_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'cancel.svg')))
|
||||
if hasattr(self, 'pause_resume_btn'):
|
||||
if self.pause_resume_btn.text() == ' Resume':
|
||||
self.pause_resume_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-play.svg')))
|
||||
else:
|
||||
self.pause_resume_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-pause.svg')))
|
||||
if hasattr(self, 'stop_fetch_btn'):
|
||||
self.stop_fetch_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'cancel.svg')))
|
||||
if hasattr(self, 'pause_fetch_btn'):
|
||||
if self.pause_fetch_btn.text() == ' Resume':
|
||||
self.pause_fetch_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-play.svg')))
|
||||
else:
|
||||
self.pause_fetch_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-pause.svg')))
|
||||
|
||||
def setup_theme_tab(self):
|
||||
pass
|
||||
|
||||
def change_theme_color(self, color, clicked_btn=None):
|
||||
pass
|
||||
|
||||
def toggle_pause_resume(self):
|
||||
if hasattr(self, 'worker'):
|
||||
icon_dir = os.path.join(os.path.dirname(__file__), 'icons')
|
||||
|
||||
if self.worker.is_paused:
|
||||
self.worker.resume()
|
||||
self.pause_resume_btn.setText(' Pause')
|
||||
self.pause_resume_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-pause.svg')))
|
||||
self.timer.start(1000)
|
||||
else:
|
||||
self.worker.pause()
|
||||
self.pause_resume_btn.setText(' Resume')
|
||||
self.pause_resume_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-play.svg')))
|
||||
|
||||
def pause_batch_fetch(self):
|
||||
self.is_auto_fetching = False
|
||||
|
||||
if hasattr(self, 'metadata_worker') and self.metadata_worker.isRunning():
|
||||
self.metadata_worker.terminate()
|
||||
self.metadata_worker.wait()
|
||||
|
||||
icon_dir = os.path.join(os.path.dirname(__file__), 'icons')
|
||||
self.pause_fetch_btn.setText(' Resume')
|
||||
self.pause_fetch_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-play.svg')))
|
||||
self.pause_fetch_btn.clicked.disconnect()
|
||||
self.pause_fetch_btn.clicked.connect(self.resume_batch_fetch)
|
||||
self.log_output.append('Batch fetching paused.')
|
||||
|
||||
def resume_batch_fetch(self):
|
||||
self.is_auto_fetching = True
|
||||
icon_dir = os.path.join(os.path.dirname(__file__), 'icons')
|
||||
self.pause_fetch_btn.setText(' Pause')
|
||||
self.pause_fetch_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-pause.svg')))
|
||||
self.pause_fetch_btn.clicked.disconnect()
|
||||
self.pause_fetch_btn.clicked.connect(self.pause_batch_fetch)
|
||||
self.log_output.append('Resuming batch fetching...')
|
||||
|
||||
if hasattr(self, 'is_multiple_user_mode') and self.is_multiple_user_mode:
|
||||
self.fetch_next_account_in_batch()
|
||||
else:
|
||||
self.fetch_next_batch_internal()
|
||||
|
||||
def stop_batch_fetch(self):
|
||||
self.is_auto_fetching = False
|
||||
self.pause_fetch_btn.hide()
|
||||
self.stop_fetch_btn.hide()
|
||||
|
||||
if hasattr(self, 'metadata_worker') and self.metadata_worker.isRunning():
|
||||
self.metadata_worker.terminate()
|
||||
self.metadata_worker.wait()
|
||||
|
||||
if self.pause_fetch_btn.text() == ' Resume':
|
||||
icon_dir = os.path.join(os.path.dirname(__file__), 'icons')
|
||||
self.pause_fetch_btn.setText(' Pause')
|
||||
self.pause_fetch_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-pause.svg')))
|
||||
self.pause_fetch_btn.clicked.disconnect()
|
||||
self.pause_fetch_btn.clicked.connect(self.pause_batch_fetch)
|
||||
|
||||
if hasattr(self, 'is_multiple_user_mode') and self.is_multiple_user_mode:
|
||||
self.log_output.append('Batch processing stopped.')
|
||||
self.is_multiple_user_mode = False
|
||||
if hasattr(self, 'accounts_to_fetch'):
|
||||
del self.accounts_to_fetch
|
||||
else:
|
||||
self.log_output.append('Auto batch mode stopped.')
|
||||
|
||||
self.enable_batch_buttons()
|
||||
|
||||
def reset_ui(self):
|
||||
super().reset_ui()
|
||||
icon_dir = os.path.join(os.path.dirname(__file__), 'icons')
|
||||
if hasattr(self, 'pause_resume_btn'):
|
||||
self.pause_resume_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-pause.svg')))
|
||||
|
||||
def on_download_finished(self, success, message):
|
||||
self.progress_bar.hide()
|
||||
self.stop_btn.hide()
|
||||
self.pause_resume_btn.hide()
|
||||
self.pause_resume_btn.setText(' Pause')
|
||||
icon_dir = os.path.join(os.path.dirname(__file__), 'icons')
|
||||
self.pause_resume_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-pause.svg')))
|
||||
self.stop_timer()
|
||||
|
||||
self.download_selected_btn.setEnabled(True)
|
||||
self.update_selected_btn.setEnabled(True)
|
||||
|
||||
self.pause_fetch_btn.hide()
|
||||
self.stop_fetch_btn.hide()
|
||||
|
||||
if success:
|
||||
self.log_output.append(f"\nStatus: {message}")
|
||||
else:
|
||||
self.log_output.append(f"Error: {message}")
|
||||
|
||||
self.tab_widget.setCurrentWidget(self.process_tab)
|
||||