This commit is contained in:
afkarxyz committed 2025-11-18 13:38:16 +07:00
1 parent f2a3810c8c
commit df0de054c8
27 files changed
+4460 -8030

No files matched your search

File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+14
View File
@@ -0,0 +1,14 @@
from core.databaseManager import DatabaseManager
from core.metadataFetcher import get_metadata, get_metadata_by_date
from core.downloadWorker import DownloadWorker
from core.metadataWorker import MetadataFetchWorker
from core.accountModel import Account
__all__ = [
'DatabaseManager',
'get_metadata',
'get_metadata_by_date',
'DownloadWorker',
'MetadataFetchWorker',
'Account'
]
+15
View File
@@ -0,0 +1,15 @@
from dataclasses import dataclass
@dataclass
class Account:
username: str
nick: str
followers: int
following: int
posts: int
media_type: str
profile_image: str = None
media_list: list = None
fetch_mode: str = 'all'
fetch_timestamp: str = None
group_id: int = None
+420
View File
@@ -0,0 +1,420 @@
import sqlite3
import os
import json
import threading
from datetime import datetime
from pathlib import Path
from typing import List, Dict, Optional, Tuple
class DatabaseManager:
def __init__(self, db_path: str):
self.db_path = db_path
self.conn = None
self._lock = threading.Lock()
self.init_database()
def init_database(self):
self.conn = sqlite3.connect(self.db_path, check_same_thread=False)
self.conn.row_factory = sqlite3.Row
self.conn.execute('PRAGMA foreign_keys = ON')
cursor = self.conn.cursor()
cursor.execute('''
CREATE TABLE IF NOT EXISTS groups (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT UNIQUE NOT NULL,
created_at TEXT
)
''')
cursor.execute('''
CREATE TABLE IF NOT EXISTS accounts (
username TEXT PRIMARY KEY COLLATE NOCASE,
nick TEXT,
followers INTEGER,
following INTEGER,
posts INTEGER,
profile_image TEXT,
fetch_timestamp TEXT,
media_type TEXT,
fetch_mode TEXT,
timeline_type TEXT DEFAULT 'media',
group_id INTEGER,
FOREIGN KEY (group_id) REFERENCES groups(id) ON DELETE SET NULL
)
''')
try:
cursor.execute("SELECT group_id FROM accounts LIMIT 1")
except sqlite3.OperationalError:
cursor.execute("ALTER TABLE accounts ADD COLUMN group_id INTEGER")
self.conn.commit()
cursor.execute('''
CREATE TABLE IF NOT EXISTS media (
id INTEGER PRIMARY KEY AUTOINCREMENT,
username TEXT COLLATE NOCASE,
tweet_id TEXT,
url TEXT,
date TEXT,
type TEXT,
FOREIGN KEY (username) REFERENCES accounts(username) ON DELETE CASCADE,
UNIQUE(username, url)
)
''')
cursor.execute('''
CREATE TABLE IF NOT EXISTS downloads (
id INTEGER PRIMARY KEY AUTOINCREMENT,
url TEXT,
filepath TEXT,
filepath_normalized TEXT COLLATE NOCASE,
username TEXT COLLATE NOCASE,
tweet_id TEXT,
download_date TEXT,
file_size INTEGER,
status TEXT,
UNIQUE(url, filepath_normalized)
)
''')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_filepath_normalized ON downloads(filepath_normalized)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_url ON downloads(url)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_username ON downloads(username)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_media_username ON media(username)')
self.conn.commit()
def save_account(self, username: str, nick: str, followers: int, following: int,
posts: int, media_type: str, profile_image: str = None,
fetch_mode: str = 'all', timeline_type: str = 'media',
fetch_timestamp: str = None, group_id: int = None):
with self._lock:
cursor = self.conn.cursor()
if fetch_timestamp is None:
fetch_timestamp = datetime.now().isoformat()
cursor.execute('''
INSERT OR REPLACE INTO accounts
(username, nick, followers, following, posts, profile_image,
fetch_timestamp, media_type, fetch_mode, timeline_type, group_id)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
''', (username, nick, followers, following, posts, profile_image,
fetch_timestamp, media_type, fetch_mode, timeline_type, group_id))
self.conn.commit()
def save_media_list(self, username: str, media_list: List[Dict]):
with self._lock:
cursor = self.conn.cursor()
cursor.execute('DELETE FROM media WHERE username = ?', (username,))
for item in media_list:
cursor.execute('''
INSERT OR IGNORE INTO media (username, tweet_id, url, date, type)
VALUES (?, ?, ?, ?, ?)
''', (username, item.get('tweet_id', ''), item['url'],
item['date'], item.get('type', 'image')))
self.conn.commit()
def get_account(self, username: str, media_type: str = 'all') -> Optional[Dict]:
cursor = self.conn.cursor()
cursor.execute('''
SELECT * FROM accounts
WHERE username = ? AND media_type = ?
''', (username, media_type))
row = cursor.fetchone()
if row:
return dict(row)
return None
def get_media_list(self, username: str) -> List[Dict]:
cursor = self.conn.cursor()
cursor.execute('''
SELECT tweet_id, url, date, type
FROM media
WHERE username = ?
ORDER BY date DESC
''', (username,))
return [dict(row) for row in cursor.fetchall()]
def get_all_accounts(self) -> List[Dict]:
cursor = self.conn.cursor()
cursor.execute('SELECT * FROM accounts ORDER BY fetch_timestamp DESC')
return [dict(row) for row in cursor.fetchall()]
def delete_account(self, username: str, media_type: str = None):
with self._lock:
cursor = self.conn.cursor()
if media_type:
cursor.execute('DELETE FROM accounts WHERE username = ? AND media_type = ?',
(username, media_type))
else:
cursor.execute('DELETE FROM accounts WHERE username = ?', (username,))
self.conn.commit()
def is_file_downloaded(self, filepath: str) -> bool:
cursor = self.conn.cursor()
filepath_normalized = filepath.lower()
cursor.execute('''
SELECT 1 FROM downloads
WHERE filepath_normalized = ? AND status = 'downloaded'
LIMIT 1
''', (filepath_normalized,))
return cursor.fetchone() is not None
def is_url_downloaded(self, url: str) -> bool:
cursor = self.conn.cursor()
cursor.execute('''
SELECT 1 FROM downloads
WHERE url = ? AND status = 'downloaded'
LIMIT 1
''', (url,))
return cursor.fetchone() is not None
def record_download(self, url: str, filepath: str, username: str,
tweet_id: str, status: str = 'downloaded', file_size: int = 0):
with self._lock:
cursor = self.conn.cursor()
filepath_normalized = filepath.lower()
download_date = datetime.now().isoformat()
cursor.execute('''
INSERT OR REPLACE INTO downloads
(url, filepath, filepath_normalized, username, tweet_id,
download_date, file_size, status)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
''', (url, filepath, filepath_normalized, username, tweet_id,
download_date, file_size, status))
self.conn.commit()
def get_download_stats(self, username: str = None) -> Dict:
cursor = self.conn.cursor()
if username:
cursor.execute('''
SELECT
COUNT(*) as total,
SUM(CASE WHEN status = 'downloaded' THEN 1 ELSE 0 END) as downloaded,
SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) as skipped,
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed,
SUM(file_size) as total_size
FROM downloads
WHERE username = ?
''', (username,))
else:
cursor.execute('''
SELECT
COUNT(*) as total,
SUM(CASE WHEN status = 'downloaded' THEN 1 ELSE 0 END) as downloaded,
SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) as skipped,
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed,
SUM(file_size) as total_size
FROM downloads
''')
row = cursor.fetchone()
return dict(row) if row else {}
def migrate_from_json(self, json_dir: str) -> Tuple[int, int]:
if not os.path.exists(json_dir):
return 0, 0
json_files = [f for f in os.listdir(json_dir) if f.endswith('.json')]
accounts_migrated = 0
media_migrated = 0
for json_file in json_files:
try:
json_path = os.path.join(json_dir, json_file)
with open(json_path, 'r', encoding='utf-8') as f:
data = json.load(f)
media_type = data.get('media_type', 'all')
if media_type not in ['all', 'image', 'video', 'gif']:
filename_parts = json_file.replace('.json', '').split('_')
if len(filename_parts) >= 1:
last_part = filename_parts[-1]
if last_part in ['all', 'image', 'video', 'gif']:
media_type = last_part
else:
media_type = 'all'
else:
media_type = 'all'
filename_parts = json_file.replace('.json', '').split('_')
if len(filename_parts) >= 2:
second_last = filename_parts[-2]
if second_last in ['media', 'timeline', 'tweets', 'with']:
timeline_type = second_last
else:
timeline_type = 'media'
else:
timeline_type = 'media'
account_info = data.get('account_info', {})
timeline = data.get('timeline', [])
username = data.get('username') or account_info.get('name', filename_parts[0] if filename_parts else 'unknown')
nick = data.get('nick') or account_info.get('nick') or account_info.get('name', username)
followers = data.get('followers') or account_info.get('followers_count', 0)
following = data.get('following') or account_info.get('friends_count', 0)
posts = data.get('posts') or account_info.get('statuses_count', 0)
profile_image = data.get('profile_image') or account_info.get('profile_image')
self.save_account(
username=username,
nick=nick,
followers=followers,
following=following,
posts=posts,
media_type=media_type,
profile_image=profile_image,
fetch_mode=data.get('fetch_mode', 'all'),
timeline_type=timeline_type
)
accounts_migrated += 1
media_list = timeline if timeline else data.get('media_list', [])
if media_list:
self.save_media_list(username, media_list)
media_migrated += len(media_list)
backup_path = json_path + '.backup'
if not os.path.exists(backup_path):
os.rename(json_path, backup_path)
except Exception as e:
print(f"Error migrating {json_file}: {e}")
continue
return accounts_migrated, media_migrated
def export_to_json(self, username: str, media_type: str, output_path: str):
account = self.get_account(username, media_type)
if not account:
return False
media_list = self.get_media_list(username)
export_data = {
'username': account['username'],
'nick': account['nick'],
'followers': account['followers'],
'following': account['following'],
'posts': account['posts'],
'profile_image': account['profile_image'],
'fetch_timestamp': account['fetch_timestamp'],
'media_type': account['media_type'],
'fetch_mode': account['fetch_mode'],
'media_list': media_list
}
with open(output_path, 'w', encoding='utf-8') as f:
json.dump(export_data, f, ensure_ascii=False, indent=2)
return True
def import_from_json(self, json_path: str) -> bool:
try:
with open(json_path, 'r', encoding='utf-8') as f:
data = json.load(f)
username = data.get('username')
if not username:
return False
self.save_account(
username=username,
nick=data.get('nick', ''),
followers=data.get('followers', 0),
following=data.get('following', 0),
posts=data.get('posts', 0),
media_type=data.get('media_type', 'all'),
profile_image=data.get('profile_image'),
fetch_mode=data.get('fetch_mode', 'all')
)
media_list = data.get('media_list', [])
if media_list:
self.save_media_list(username, media_list)
return True
except Exception as e:
print(f"Error importing JSON: {e}")
return False
def close(self):
if self.conn:
self.conn.close()
def create_group(self, name: str) -> Optional[int]:
with self._lock:
try:
cursor = self.conn.cursor()
created_at = datetime.now().isoformat()
cursor.execute('INSERT INTO groups (name, created_at) VALUES (?, ?)', (name, created_at))
self.conn.commit()
return cursor.lastrowid
except sqlite3.IntegrityError:
return None
def get_all_groups(self) -> List[Dict]:
cursor = self.conn.cursor()
cursor.execute('SELECT * FROM groups ORDER BY name')
return [dict(row) for row in cursor.fetchall()]
def get_group(self, group_id: int) -> Optional[Dict]:
cursor = self.conn.cursor()
cursor.execute('SELECT * FROM groups WHERE id = ?', (group_id,))
row = cursor.fetchone()
return dict(row) if row else None
def update_group(self, group_id: int, name: str) -> bool:
with self._lock:
try:
cursor = self.conn.cursor()
cursor.execute('UPDATE groups SET name = ? WHERE id = ?', (name, group_id))
self.conn.commit()
return True
except sqlite3.IntegrityError:
return False
def delete_group(self, group_id: int):
with self._lock:
cursor = self.conn.cursor()
cursor.execute('UPDATE accounts SET group_id = NULL WHERE group_id = ?', (group_id,))
cursor.execute('DELETE FROM groups WHERE id = ?', (group_id,))
self.conn.commit()
def assign_account_to_group(self, username: str, group_id: Optional[int]):
with self._lock:
cursor = self.conn.cursor()
cursor.execute('UPDATE accounts SET group_id = ? WHERE username = ?', (group_id, username))
self.conn.commit()
def get_accounts_by_group(self, group_id: Optional[int]) -> List[Dict]:
cursor = self.conn.cursor()
if group_id is None:
cursor.execute('SELECT * FROM accounts WHERE group_id IS NULL ORDER BY fetch_timestamp DESC')
else:
cursor.execute('SELECT * FROM accounts WHERE group_id = ? ORDER BY fetch_timestamp DESC', (group_id,))
return [dict(row) for row in cursor.fetchall()]
+307
View File
@@ -0,0 +1,307 @@
import sys
import os
import asyncio
import aiohttp
import subprocess
import imageio_ffmpeg
from datetime import datetime
from PyQt6.QtCore import QThread, pyqtSignal
DEFAULT_BATCH_SIZE = 25
TIMEOUT_SECONDS_MAC = 60
TIMEOUT_SECONDS_DEFAULT = 30
BATCH_SLEEP_INTERVAL = 0.1
PAUSE_CHECK_INTERVAL = 0.1
class DownloadWorker(QThread):
finished = pyqtSignal(bool, str)
progress = pyqtSignal(str, int)
conversion_progress = pyqtSignal(str, int)
download_progress = pyqtSignal(str, int)
def __init__(self, accounts, outpath, auth_token, filename_format='username_date',
download_batch_size=DEFAULT_BATCH_SIZE, convert_gif=False, gif_resolution='original', gif_conversion_mode='better', db_manager=None):
super().__init__()
self.accounts = accounts
self.outpath = outpath
self.auth_token = auth_token
self.filename_format = filename_format
self.download_batch_size = download_batch_size
self.convert_gif = convert_gif
self.gif_resolution = gif_resolution
self.gif_conversion_mode = gif_conversion_mode
self.is_paused = False
self.is_stopped = False
self.filepath_map = []
self.db_manager = db_manager
async def download_file(self, session, url, filepath, username, tweet_id):
try:
if self.db_manager and self.db_manager.is_url_downloaded(url):
return True, True
if os.path.exists(filepath):
if self.db_manager:
file_size = os.path.getsize(filepath)
self.db_manager.record_download(url, filepath, username, tweet_id, 'downloaded', file_size)
return True, True
if self.is_stopped:
return False, False
try:
os.makedirs(os.path.dirname(filepath), exist_ok=True)
except OSError as e:
return False, False
async with session.get(url) as response:
if response.status == 200:
content = await response.read()
try:
with open(filepath, 'wb') as f:
f.write(content)
except (IOError, OSError) as e:
if self.db_manager:
self.db_manager.record_download(url, filepath, username, tweet_id, 'failed', 0)
return False, False
if self.db_manager:
self.db_manager.record_download(url, filepath, username, tweet_id, 'downloaded', len(content))
return True, False
return False, False
except asyncio.TimeoutError:
if self.db_manager:
self.db_manager.record_download(url, filepath, username, tweet_id, 'failed', 0)
return False, False
except aiohttp.ClientError as e:
if self.db_manager:
self.db_manager.record_download(url, filepath, username, tweet_id, 'failed', 0)
return False, False
except Exception as e:
if self.db_manager:
self.db_manager.record_download(url, filepath, username, tweet_id, 'failed', 0)
return False, False
async def download_account_media(self, account):
if not account.media_list:
return 0, 0, 0
safe_username = os.path.basename(account.username.replace('/', '_').replace('\\', '_'))
account_output_dir = os.path.join(self.outpath, safe_username)
os.makedirs(account_output_dir, exist_ok=True)
timeout_seconds = TIMEOUT_SECONDS_MAC if sys.platform == 'darwin' else TIMEOUT_SECONDS_DEFAULT
timeout = aiohttp.ClientTimeout(total=timeout_seconds)
connector = aiohttp.TCPConnector(limit=self.download_batch_size)
async with aiohttp.ClientSession(timeout=timeout, connector=connector) as session:
total = len(account.media_list)
completed = 0
skipped = 0
failed = 0
used_filenames = set()
for i in range(0, total, self.download_batch_size):
if self.is_stopped:
break
while self.is_paused:
if self.is_stopped:
return completed, skipped, failed
await asyncio.sleep(PAUSE_CHECK_INTERVAL)
batch = account.media_list[i:i + self.download_batch_size]
tasks = []
for item in batch:
url = item['url']
date = datetime.strptime(item['date'], "%Y-%m-%d %H:%M:%S")
formatted_date = date.strftime("%Y%m%d_%H%M%S")
tweet_id = str(item.get('tweet_id', ''))
item_type = item.get('type', '')
if item_type == 'animated_gif':
media_type_folder = 'gif'
extension = 'mp4'
elif item_type == 'video' or 'video.twimg.com' in url:
media_type_folder = 'video'
extension = 'mp4'
else:
media_type_folder = 'image'
extension = 'jpg'
media_output_dir = os.path.join(account_output_dir, media_type_folder)
if self.filename_format == "username_date":
base_filename = f"{safe_username}_{formatted_date}_{tweet_id}"
else:
base_filename = f"{formatted_date}_{safe_username}_{tweet_id}"
filename = f"{base_filename}.{extension}"
counter = 1
while filename in used_filenames:
filename = f"{base_filename}_{counter:02d}.{extension}"
counter += 1
used_filenames.add(filename)
filepath = os.path.join(media_output_dir, filename)
self.filepath_map.append((item, filepath))
task = asyncio.create_task(self.download_file(session, url, filepath, account.username, tweet_id))
tasks.append(task)
results = await asyncio.gather(*tasks, return_exceptions=True)
for result in results:
if isinstance(result, tuple):
success, was_skipped = result
if success:
if was_skipped:
skipped += 1
else:
completed += 1
else:
failed += 1
else:
failed += 1
progress_percent = int((completed + skipped + failed) / total * 100)
media_type_display = account.media_type if account.media_type != 'all' else 'media'
self.download_progress.emit(f"Downloading {account.username}'s {media_type_display}: {completed + skipped + failed:,}/{total:,}", progress_percent)
await asyncio.sleep(BATCH_SLEEP_INTERVAL)
return completed, skipped, failed
def run(self):
try:
if sys.platform == 'darwin':
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
total_accounts = len(self.accounts)
overall_completed = 0
overall_skipped = 0
overall_failed = 0
for i, account in enumerate(self.accounts):
if self.is_stopped:
break
while self.is_paused:
if self.is_stopped:
return
self.msleep(100)
media_type_display = account.media_type if account.media_type != 'all' else 'media'
self.progress.emit(f"Downloading from account: {account.username} ({media_type_display}) - ({i+1}/{total_accounts})",
int((i) / total_accounts * 100))
completed, skipped, failed = asyncio.run(self.download_account_media(account))
overall_completed += completed
overall_skipped += skipped
overall_failed += failed
self.progress.emit(f"Account {account.username} ({media_type_display}): {completed:,} downloaded, {skipped:,} skipped, {failed:,} failed",
int((i + 1) / total_accounts * 100))
if not self.is_stopped:
if self.convert_gif:
try:
ffmpeg_exe = imageio_ffmpeg.get_ffmpeg_exe()
gif_items = [(item, fp) for item, fp in self.filepath_map if item.get('type') == 'animated_gif']
total_gifs = len(gif_items)
if total_gifs > 0:
if sys.platform == 'win32':
creationflags = subprocess.CREATE_NO_WINDOW
else:
creationflags = 0
converted_count = 0
skipped_count = 0
self.progress.emit("Starting GIF conversion...", 0)
for idx, (item, fp) in enumerate(gif_items, start=1):
if self.is_stopped:
break
while self.is_paused:
if self.is_stopped:
break
self.msleep(100)
if self.is_stopped:
break
gif_fp = fp.rsplit('.', 1)[0] + '.gif'
if os.path.exists(gif_fp):
try:
os.remove(fp)
skipped_count += 1
conv_progress = int((idx / total_gifs) * 100)
quality_display = f"({self.gif_conversion_mode}) - ({self.gif_resolution})"
self.conversion_progress.emit(f"Converting GIF {idx:,}/{total_gifs:,} {quality_display} - Skipped (exists)", conv_progress)
continue
except Exception:
pass
if self.gif_conversion_mode == 'fast':
ffmpeg_args = [ffmpeg_exe, '-i', fp, '-y', gif_fp]
else:
if self.gif_resolution == 'high':
ffmpeg_args = [ffmpeg_exe, '-i', fp, '-lavfi', 'scale=800:-1:flags=lanczos,palettegen=stats_mode=full[palette];[0:v]scale=800:-1:flags=lanczos[scaled];[scaled][palette]paletteuse=dither=sierra2_4a', '-r', '15', '-y', gif_fp]
elif self.gif_resolution == 'medium':
ffmpeg_args = [ffmpeg_exe, '-i', fp, '-lavfi', 'scale=600:-1:flags=lanczos,palettegen=stats_mode=full[palette];[0:v]scale=600:-1:flags=lanczos[scaled];[scaled][palette]paletteuse=dither=sierra2_4a', '-r', '10', '-y', gif_fp]
elif self.gif_resolution == 'low':
ffmpeg_args = [ffmpeg_exe, '-i', fp, '-lavfi', 'scale=400:-1:flags=lanczos,palettegen=stats_mode=full[palette];[0:v]scale=400:-1:flags=lanczos[scaled];[scaled][palette]paletteuse=dither=sierra2_4a', '-r', '8', '-y', gif_fp]
else:
ffmpeg_args = [ffmpeg_exe, '-i', fp, '-lavfi', 'palettegen=stats_mode=full[palette];[0:v][palette]paletteuse=dither=sierra2_4a', '-y', gif_fp]
try:
if sys.platform == 'win32':
result = subprocess.run(ffmpeg_args, capture_output=True, creationflags=creationflags)
else:
result = subprocess.run(ffmpeg_args, capture_output=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
if result.returncode == 0 and os.path.exists(gif_fp):
try:
os.remove(fp)
converted_count += 1
except Exception:
pass
elif result.returncode != 0:
error_msg = result.stderr.decode('utf-8', errors='ignore') if result.stderr else 'Unknown error'
self.progress.emit(f"GIF conversion warning for file {idx}: {error_msg[:100]}", 0)
except Exception as e:
self.progress.emit(f"GIF conversion error for file {idx}: {str(e)[:100]}", 0)
conv_progress = int((idx / total_gifs) * 100)
quality_display = f"({self.gif_conversion_mode}) - ({self.gif_resolution})"
self.conversion_progress.emit(f"Converting GIF {idx:,}/{total_gifs:,} {quality_display}", conv_progress)
if converted_count > 0 or skipped_count > 0:
completion_msg = f"GIF conversion completed: {converted_count:,} converted"
if skipped_count > 0:
completion_msg += f", {skipped_count:,} skipped (already exists)"
self.progress.emit(completion_msg, 100)
else:
self.progress.emit("GIF conversion completed", 100)
except Exception as conv_e:
self.progress.emit(f"GIF conversion error: {conv_e}", 0)
success_message = f"Download completed! {overall_completed:,} files downloaded"
if overall_skipped > 0:
success_message += f", {overall_skipped:,} skipped"
if overall_failed > 0:
success_message += f", {overall_failed:,} failed"
self.finished.emit(True, success_message)
except Exception as e:
self.finished.emit(False, str(e))
def pause(self):
self.is_paused = True
def resume(self):
self.is_paused = False
def stop(self):
self.is_stopped = True
self.is_paused = False
+145 -2
View File
@@ -3,7 +3,143 @@ import json
from datetime import datetime
from gallery_dl.extractor import twitter
def get_metadata(username, auth_token, timeline_type="timeline", batch_size=0, page=0, media_type="all"):
def get_metadata_by_date(username, auth_token, date_start, date_end, media_filter="filter:media", output_file=None):
query = f"from:{username} since:{date_start} until:{date_end}"
if media_filter:
query += f" {media_filter}"
url = f"https://x.com/search?q={query}"
extractor_class = twitter.TwitterSearchExtractor
match = re.match(extractor_class.pattern, url)
if not match:
raise ValueError(f"Invalid search URL: {url}")
extractor = extractor_class(match)
config_dict = {
"cookies": {
"auth_token": auth_token
},
"retweets": False
}
extractor.config = lambda key, default=None: config_dict.get(key, default)
try:
extractor.initialize()
api = twitter.TwitterAPI(extractor)
try:
user = api.user_by_screen_name(username)
if "legacy" in user and user["legacy"].get("withheld_scope"):
raise ValueError("withheld")
except Exception as e:
error_msg = str(e).lower()
if "withheld" in error_msg or (hasattr(e, "response") and "withheld" in str(e.response.text).lower()):
raise ValueError("withheld")
raise
user_data = extractor._transform_user(user)
structured_output = {
'account_info': {
'name': user_data.get('name', ''),
'nick': user_data.get('nick', ''),
'date': user_data.get('date', ''),
'followers_count': user_data.get('followers_count', 0),
'friends_count': user_data.get('friends_count', 0),
'profile_image': user_data.get('profile_image', ''),
'statuses_count': user_data.get('statuses_count', 0)
},
'total_urls': 0,
'timeline': [],
'search_query': query,
'date_filter': {
'start': date_start,
'end': date_end,
'method': 'search_api'
}
}
if isinstance(structured_output['account_info']['date'], datetime):
structured_output['account_info']['date'] = structured_output['account_info']['date'].strftime("%Y-%m-%d %H:%M:%S")
new_timeline_entries = []
try:
iterator = iter(extractor)
while True:
try:
item = next(iterator)
if isinstance(item, tuple) and len(item) >= 3:
media_url = item[1]
tweet_data = item[2]
if 'pbs.twimg.com' in media_url or 'video.twimg.com' in media_url:
tweet_date = tweet_data.get('date', datetime.now())
if isinstance(tweet_date, datetime):
tweet_date = tweet_date.strftime("%Y-%m-%d %H:%M:%S")
timeline_entry = {
'url': media_url,
'date': tweet_date,
'tweet_id': tweet_data.get('tweet_id', 0),
}
if 'type' in tweet_data:
timeline_entry['type'] = tweet_data['type']
if 'retweet_id' in tweet_data and tweet_data['retweet_id']:
timeline_entry['retweet_id'] = tweet_data['retweet_id']
timeline_entry['is_retweet'] = True
else:
timeline_entry['is_retweet'] = False
new_timeline_entries.append(timeline_entry)
structured_output['total_urls'] += 1
except StopIteration:
break
except Exception as e:
pass
structured_output['timeline'] = new_timeline_entries
structured_output['metadata'] = {
"new_entries": len(new_timeline_entries),
"method": "search_api",
"date_range": f"{date_start} to {date_end}"
}
if output_file:
try:
with open(output_file, 'w', encoding='utf-8') as f:
json.dump(structured_output, f, ensure_ascii=False, indent=2)
except Exception as e:
pass
return structured_output
except Exception as e:
error_msg = str(e).lower()
if "withheld" in error_msg or e.__class__.__name__ == "ValueError" and str(e) == "withheld":
return {"error": "To download withheld accounts, use this userscript version: https://www.patreon.com/exyezed"}
else:
error_str = str(e)
if error_str == "None":
return {"error": "Failed to authenticate. Please verify your auth token is valid and not expired."}
else:
return {"error": error_str}
def get_metadata(username, auth_token, timeline_type="timeline", batch_size=0, page=0, media_type="all", retweets=False):
url = f"https://x.com/{username}/{timeline_type}"
if timeline_type == "media":
@@ -24,7 +160,8 @@ def get_metadata(username, auth_token, timeline_type="timeline", batch_size=0, p
config_dict = {
"cookies": {
"auth_token": auth_token
}
},
"retweets": retweets
}
if batch_size > 0:
@@ -117,6 +254,12 @@ def get_metadata(username, auth_token, timeline_type="timeline", batch_size=0, p
if 'type' in tweet_data:
timeline_entry['type'] = tweet_data['type']
if 'retweet_id' in tweet_data and tweet_data['retweet_id']:
timeline_entry['retweet_id'] = tweet_data['retweet_id']
timeline_entry['is_retweet'] = True
else:
timeline_entry['is_retweet'] = False
if media_type == 'all' or (
(media_type == 'image' and 'pbs.twimg.com' in media_url and tweet_data.get('type') == 'photo') or
(media_type == 'video' and 'video.twimg.com' in media_url and tweet_data.get('type') == 'video') or
+60
View File
@@ -0,0 +1,60 @@
from PyQt6.QtCore import QThread, pyqtSignal
from core.metadataFetcher import get_metadata, get_metadata_by_date
class MetadataFetchWorker(QThread):
finished = pyqtSignal(dict)
error = pyqtSignal(str)
def __init__(self, username, media_type='all', batch_mode=False, batch_size=0, page=0, timeline_type='media', include_retweets=False, use_date_range=False, date_start=None, date_end=None):
super().__init__()
self.username = username
self.media_type = media_type
self.batch_mode = batch_mode
self.batch_size = batch_size
self.page = page
self.timeline_type = timeline_type
self.include_retweets = include_retweets
self.use_date_range = use_date_range
self.date_start = date_start
self.date_end = date_end
self.auth_token = None
def normalize_url(self, url_or_username):
url_or_username = url_or_username.strip()
username = url_or_username
if "x.com/" in url_or_username or "twitter.com/" in url_or_username:
parts = url_or_username.split('/')
for i, part in enumerate(parts):
if part in ['x.com', 'twitter.com'] and i + 1 < len(parts):
username = parts[i + 1]
username = username.split('/')[0]
break
username = username.strip()
return username
def run(self):
try:
normalized = self.normalize_url(self.username)
if self.use_date_range and self.date_start and self.date_end:
data = get_metadata_by_date(
username=normalized,
auth_token=self.auth_token,
date_start=self.date_start,
date_end=self.date_end,
media_filter="filter:media" if self.media_type == 'all' else f"filter:{self.media_type}"
)
else:
data = get_metadata(
username=normalized,
auth_token=self.auth_token,
timeline_type=self.timeline_type,
batch_size=self.batch_size if self.batch_mode else 0,
page=self.page,
media_type=self.media_type,
retweets=self.include_retweets
)
self.finished.emit(data)
except Exception as e:
self.error.emit(str(e))
+4 -2
View File
@@ -1,6 +1,8 @@
PyQt6
PyQt6==6.9.1
requests
aiohttp
imageio-ffmpeg
packaging
gallery-dl
gallery-dl
pyqtdarktheme
pyinstaller
+7
View File
@@ -0,0 +1,7 @@
from ui.mainWindow import TwitterXMediaBatchDownloaderGUI
from ui.dialogs import UpdateDialog
__all__ = [
'TwitterXMediaBatchDownloaderGUI',
'UpdateDialog'
]
+33
View File
@@ -0,0 +1,33 @@
from PyQt6.QtWidgets import (
QDialog, QVBoxLayout, QLabel, QPushButton, QDialogButtonBox
)
from PyQt6.QtCore import Qt
class UpdateDialog(QDialog):
def __init__(self, current_version, new_version, parent=None):
super().__init__(parent)
self.setWindowTitle("Update Now")
self.setFixedWidth(400)
self.setModal(True)
layout = QVBoxLayout()
message = QLabel(f"Twitter/X Media Batch Downloader v{new_version} Available!")
message.setWordWrap(True)
layout.addWidget(message)
button_box = QDialogButtonBox()
self.update_button = QPushButton("Check")
self.update_button.setCursor(Qt.CursorShape.PointingHandCursor)
self.cancel_button = QPushButton("Later")
self.cancel_button.setCursor(Qt.CursorShape.PointingHandCursor)
button_box.addButton(self.update_button, QDialogButtonBox.ButtonRole.AcceptRole)
button_box.addButton(self.cancel_button, QDialogButtonBox.ButtonRole.RejectRole)
layout.addWidget(button_box)
self.setLayout(layout)
self.update_button.clicked.connect(self.accept)
self.cancel_button.clicked.connect(self.reject)
+1
View File
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round" class="icon icon-tabler icons-tabler-outline icon-tabler-cancel"><path stroke="none" d="M0 0h24v24H0z" fill="none"/><path d="M3 12a9 9 0 1 0 18 0a9 9 0 1 0 -18 0" /><path d="M18.364 5.636l-12.728 12.728" /></svg>

After

Width:  |  Height:  |  Size: 396 B

File renamed without changes.

Before

Width:  |  Height:  |  Size: 616 B

After

Width:  |  Height:  |  Size: 616 B

File renamed without changes.

Before

Width:  |  Height:  |  Size: 610 B

After

Width:  |  Height:  |  Size: 610 B

+1
View File
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round" class="icon icon-tabler icons-tabler-outline icon-tabler-device-floppy"><path stroke="none" d="M0 0h24v24H0z" fill="none"/><path d="M6 4h10l4 4v10a2 2 0 0 1 -2 2h-12a2 2 0 0 1 -2 -2v-12a2 2 0 0 1 2 -2" /><path d="M12 14m-2 0a2 2 0 1 0 4 0a2 2 0 1 0 -4 0" /><path d="M14 4l0 4l-6 0l0 -4" /></svg>

After

Width:  |  Height:  |  Size: 479 B

File renamed without changes.

Before

Width:  |  Height:  |  Size: 423 B

After

Width:  |  Height:  |  Size: 423 B

+1
View File
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round" class="icon icon-tabler icons-tabler-outline icon-tabler-edit"><path stroke="none" d="M0 0h24v24H0z" fill="none"/><path d="M7 7h-1a2 2 0 0 0 -2 2v9a2 2 0 0 0 2 2h9a2 2 0 0 0 2 -2v-1" /><path d="M20.385 6.585a2.1 2.1 0 0 0 -2.97 -2.97l-8.415 8.385v3h3l8.385 -8.415z" /><path d="M16 5l3 3" /></svg>

After

Width:  |  Height:  |  Size: 480 B

File renamed without changes.
File renamed without changes.

Before

Width:  |  Height:  |  Size: 173 KiB

After

Width:  |  Height:  |  Size: 173 KiB

File renamed without changes.

Before

Width:  |  Height:  |  Size: 865 B

After

Width:  |  Height:  |  Size: 865 B

+1
View File
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round" class="icon icon-tabler icons-tabler-outline icon-tabler-player-pause"><path stroke="none" d="M0 0h24v24H0z" fill="none"/><path d="M6 5m0 1a1 1 0 0 1 1 -1h2a1 1 0 0 1 1 1v12a1 1 0 0 1 -1 1h-2a1 1 0 0 1 -1 -1z" /><path d="M14 5m0 1a1 1 0 0 1 1 -1h2a1 1 0 0 1 1 1v12a1 1 0 0 1 -1 1h-2a1 1 0 0 1 -1 -1z" /></svg>

After

Width:  |  Height:  |  Size: 493 B

+1
View File
@@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round" class="icon icon-tabler icons-tabler-outline icon-tabler-player-play"><path stroke="none" d="M0 0h24v24H0z" fill="none"/><path d="M7 4v16l13 -8z" /></svg>

After

Width:  |  Height:  |  Size: 338 B

File renamed without changes.

Before

Width:  |  Height:  |  Size: 423 B

After

Width:  |  Height:  |  Size: 423 B

File renamed without changes.

Before

Width:  |  Height:  |  Size: 495 B

After

Width:  |  Height:  |  Size: 495 B

+3168
View File
File diff suppressed because it is too large. Load diff
+267
View File
@@ -0,0 +1,267 @@
import os
from PyQt6.QtWidgets import QApplication, QLabel
from PyQt6.QtCore import Qt, QSize, QEvent
from PyQt6.QtGui import QIcon, QPalette, QColor
from ui.mainWindow import TwitterXMediaBatchDownloaderGUI
class TwitterXMediaBatchDownloaderGUI_Unix(TwitterXMediaBatchDownloaderGUI):
def __init__(self):
super().__init__()
self.setup_unix_specific()
QApplication.instance().installEventFilter(self)
def eventFilter(self, obj, event):
if event.type() == QEvent.Type.ApplicationPaletteChange:
self.update_button_icons_unix()
self.update_ui_colors()
return super().eventFilter(obj, event)
def update_ui_colors(self):
from PyQt6.QtWidgets import QLineEdit, QTextEdit, QComboBox, QCheckBox, QPushButton, QProgressBar, QDateEdit, QListWidget, QTabWidget
new_palette = QApplication.palette()
self.account_list.setStyleSheet("""
QListWidget {
background-color: palette(base);
padding: 4px;
outline: none;
}
QListWidget::item {
padding: 3px 6px;
margin: 0px 0px;
border: none;
outline: none;
}
QListWidget::item:selected {
background-color: palette(highlight);
border: none;
outline: none;
}
QListWidget::item:focus {
border: none;
outline: none;
}
QListWidget::item:hover {
background-color: palette(highlight);
}
""")
widget_types = [QLabel, QLineEdit, QTextEdit, QComboBox, QCheckBox, QPushButton, QProgressBar, QDateEdit, QListWidget, QTabWidget]
for widget_type in widget_types:
for widget in self.findChildren(widget_type):
widget.setPalette(new_palette)
widget.update()
for i in range(self.account_list.count()):
item = self.account_list.item(i)
if item:
self.account_list.update(self.account_list.indexFromItem(item))
self.setPalette(new_palette)
self.update()
def setup_unix_specific(self):
self.setFixedWidth(670)
if hasattr(self, 'fetch_btn'):
self.fetch_btn.setFixedWidth(100)
if hasattr(self, 'start_date_edit'):
self.start_date_edit.setMinimumWidth(130)
if hasattr(self, 'end_date_edit'):
self.end_date_edit.setMinimumWidth(130)
if hasattr(self, 'output_browse'):
self.output_browse.setFixedWidth(100)
self.account_list.setStyleSheet("""
QListWidget {
background-color: palette(base);
padding: 4px;
outline: none;
}
QListWidget::item {
padding: 3px 6px;
margin: 0px 0px;
border: none;
outline: none;
}
QListWidget::item:selected {
background-color: palette(highlight);
border: none;
outline: none;
}
QListWidget::item:focus {
border: none;
outline: none;
}
QListWidget::item:hover {
background-color: palette(highlight);
}
""")
self.update_button_icons_unix()
def create_colored_icon(self, icon_path, color=None):
if not os.path.exists(icon_path):
return QIcon()
palette = QApplication.palette()
text_color = palette.color(QPalette.ColorRole.Text)
color_hex = text_color.name()
from PyQt6.QtSvg import QSvgRenderer
from PyQt6.QtGui import QPixmap, QPainter
renderer = QSvgRenderer(icon_path)
pixmap = QPixmap(24, 24)
pixmap.fill(Qt.GlobalColor.transparent)
painter = QPainter(pixmap)
renderer.render(painter)
painter.setCompositionMode(QPainter.CompositionMode.CompositionMode_SourceIn)
painter.fillRect(pixmap.rect(), QColor(color_hex))
painter.end()
return QIcon(pixmap)
def update_button_icons(self):
self.update_button_icons_unix()
def update_button_icons_unix_dashboard(self):
if not hasattr(self, 'import_btn'):
return
icon_dir = os.path.join(os.path.dirname(__file__), 'icons')
self.import_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'database-import.svg')))
self.export_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'database-export.svg')))
self.download_selected_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'download.svg')))
self.update_selected_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'reload.svg')))
self.delete_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'trash.svg')))
def update_button_icons_unix(self):
self.update_button_icons_unix_dashboard()
icon_dir = os.path.join(os.path.dirname(__file__), 'icons')
if hasattr(self, 'stop_btn'):
self.stop_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'cancel.svg')))
if hasattr(self, 'pause_resume_btn'):
if self.pause_resume_btn.text() == ' Resume':
self.pause_resume_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-play.svg')))
else:
self.pause_resume_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-pause.svg')))
if hasattr(self, 'stop_fetch_btn'):
self.stop_fetch_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'cancel.svg')))
if hasattr(self, 'pause_fetch_btn'):
if self.pause_fetch_btn.text() == ' Resume':
self.pause_fetch_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-play.svg')))
else:
self.pause_fetch_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-pause.svg')))
def setup_theme_tab(self):
pass
def change_theme_color(self, color, clicked_btn=None):
pass
def toggle_pause_resume(self):
if hasattr(self, 'worker'):
icon_dir = os.path.join(os.path.dirname(__file__), 'icons')
if self.worker.is_paused:
self.worker.resume()
self.pause_resume_btn.setText(' Pause')
self.pause_resume_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-pause.svg')))
self.timer.start(1000)
else:
self.worker.pause()
self.pause_resume_btn.setText(' Resume')
self.pause_resume_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-play.svg')))
def pause_batch_fetch(self):
self.is_auto_fetching = False
if hasattr(self, 'metadata_worker') and self.metadata_worker.isRunning():
self.metadata_worker.terminate()
self.metadata_worker.wait()
icon_dir = os.path.join(os.path.dirname(__file__), 'icons')
self.pause_fetch_btn.setText(' Resume')
self.pause_fetch_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-play.svg')))
self.pause_fetch_btn.clicked.disconnect()
self.pause_fetch_btn.clicked.connect(self.resume_batch_fetch)
self.log_output.append('Batch fetching paused.')
def resume_batch_fetch(self):
self.is_auto_fetching = True
icon_dir = os.path.join(os.path.dirname(__file__), 'icons')
self.pause_fetch_btn.setText(' Pause')
self.pause_fetch_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-pause.svg')))
self.pause_fetch_btn.clicked.disconnect()
self.pause_fetch_btn.clicked.connect(self.pause_batch_fetch)
self.log_output.append('Resuming batch fetching...')
if hasattr(self, 'is_multiple_user_mode') and self.is_multiple_user_mode:
self.fetch_next_account_in_batch()
else:
self.fetch_next_batch_internal()
def stop_batch_fetch(self):
self.is_auto_fetching = False
self.pause_fetch_btn.hide()
self.stop_fetch_btn.hide()
if hasattr(self, 'metadata_worker') and self.metadata_worker.isRunning():
self.metadata_worker.terminate()
self.metadata_worker.wait()
if self.pause_fetch_btn.text() == ' Resume':
icon_dir = os.path.join(os.path.dirname(__file__), 'icons')
self.pause_fetch_btn.setText(' Pause')
self.pause_fetch_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-pause.svg')))
self.pause_fetch_btn.clicked.disconnect()
self.pause_fetch_btn.clicked.connect(self.pause_batch_fetch)
if hasattr(self, 'is_multiple_user_mode') and self.is_multiple_user_mode:
self.log_output.append('Batch processing stopped.')
self.is_multiple_user_mode = False
if hasattr(self, 'accounts_to_fetch'):
del self.accounts_to_fetch
else:
self.log_output.append('Auto batch mode stopped.')
self.enable_batch_buttons()
def reset_ui(self):
super().reset_ui()
icon_dir = os.path.join(os.path.dirname(__file__), 'icons')
if hasattr(self, 'pause_resume_btn'):
self.pause_resume_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-pause.svg')))
def on_download_finished(self, success, message):
self.progress_bar.hide()
self.stop_btn.hide()
self.pause_resume_btn.hide()
self.pause_resume_btn.setText(' Pause')
icon_dir = os.path.join(os.path.dirname(__file__), 'icons')
self.pause_resume_btn.setIcon(self.create_colored_icon(os.path.join(icon_dir, 'player-pause.svg')))
self.stop_timer()
self.download_selected_btn.setEnabled(True)
self.update_selected_btn.setEnabled(True)
self.pause_fetch_btn.hide()
self.stop_fetch_btn.hide()
if success:
self.log_output.append(f"\nStatus: {message}")
else:
self.log_output.append(f"Error: {message}")
self.tab_widget.setCurrentWidget(self.process_tab)