mirror of
https://github.com/Alvin-Zilverstand/Twitter-X-Media-Batch-Downloader.git
synced 2026-11-21 13:53:13 +01:00
421 lines
16 KiB
Python
421 lines
16 KiB
Python
import sqlite3
|
|
import os
|
|
import json
|
|
import threading
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import List, Dict, Optional, Tuple
|
|
|
|
class DatabaseManager:
|
|
def __init__(self, db_path: str):
|
|
self.db_path = db_path
|
|
self.conn = None
|
|
self._lock = threading.Lock()
|
|
self.init_database()
|
|
|
|
def init_database(self):
|
|
self.conn = sqlite3.connect(self.db_path, check_same_thread=False)
|
|
self.conn.row_factory = sqlite3.Row
|
|
|
|
self.conn.execute('PRAGMA foreign_keys = ON')
|
|
|
|
cursor = self.conn.cursor()
|
|
|
|
cursor.execute('''
|
|
CREATE TABLE IF NOT EXISTS groups (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
name TEXT UNIQUE NOT NULL,
|
|
created_at TEXT
|
|
)
|
|
''')
|
|
|
|
cursor.execute('''
|
|
CREATE TABLE IF NOT EXISTS accounts (
|
|
username TEXT PRIMARY KEY COLLATE NOCASE,
|
|
nick TEXT,
|
|
followers INTEGER,
|
|
following INTEGER,
|
|
posts INTEGER,
|
|
profile_image TEXT,
|
|
fetch_timestamp TEXT,
|
|
media_type TEXT,
|
|
fetch_mode TEXT,
|
|
timeline_type TEXT DEFAULT 'media',
|
|
group_id INTEGER,
|
|
FOREIGN KEY (group_id) REFERENCES groups(id) ON DELETE SET NULL
|
|
)
|
|
''')
|
|
|
|
try:
|
|
cursor.execute("SELECT group_id FROM accounts LIMIT 1")
|
|
except sqlite3.OperationalError:
|
|
cursor.execute("ALTER TABLE accounts ADD COLUMN group_id INTEGER")
|
|
self.conn.commit()
|
|
|
|
cursor.execute('''
|
|
CREATE TABLE IF NOT EXISTS media (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
username TEXT COLLATE NOCASE,
|
|
tweet_id TEXT,
|
|
url TEXT,
|
|
date TEXT,
|
|
type TEXT,
|
|
FOREIGN KEY (username) REFERENCES accounts(username) ON DELETE CASCADE,
|
|
UNIQUE(username, url)
|
|
)
|
|
''')
|
|
|
|
cursor.execute('''
|
|
CREATE TABLE IF NOT EXISTS downloads (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
url TEXT,
|
|
filepath TEXT,
|
|
filepath_normalized TEXT COLLATE NOCASE,
|
|
username TEXT COLLATE NOCASE,
|
|
tweet_id TEXT,
|
|
download_date TEXT,
|
|
file_size INTEGER,
|
|
status TEXT,
|
|
UNIQUE(url, filepath_normalized)
|
|
)
|
|
''')
|
|
|
|
cursor.execute('CREATE INDEX IF NOT EXISTS idx_filepath_normalized ON downloads(filepath_normalized)')
|
|
cursor.execute('CREATE INDEX IF NOT EXISTS idx_url ON downloads(url)')
|
|
cursor.execute('CREATE INDEX IF NOT EXISTS idx_username ON downloads(username)')
|
|
cursor.execute('CREATE INDEX IF NOT EXISTS idx_media_username ON media(username)')
|
|
|
|
self.conn.commit()
|
|
|
|
def save_account(self, username: str, nick: str, followers: int, following: int,
|
|
posts: int, media_type: str, profile_image: str = None,
|
|
fetch_mode: str = 'all', timeline_type: str = 'media',
|
|
fetch_timestamp: str = None, group_id: int = None):
|
|
with self._lock:
|
|
cursor = self.conn.cursor()
|
|
|
|
if fetch_timestamp is None:
|
|
fetch_timestamp = datetime.now().isoformat()
|
|
|
|
cursor.execute('''
|
|
INSERT OR REPLACE INTO accounts
|
|
(username, nick, followers, following, posts, profile_image,
|
|
fetch_timestamp, media_type, fetch_mode, timeline_type, group_id)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
''', (username, nick, followers, following, posts, profile_image,
|
|
fetch_timestamp, media_type, fetch_mode, timeline_type, group_id))
|
|
|
|
self.conn.commit()
|
|
|
|
def save_media_list(self, username: str, media_list: List[Dict]):
|
|
with self._lock:
|
|
cursor = self.conn.cursor()
|
|
|
|
cursor.execute('DELETE FROM media WHERE username = ?', (username,))
|
|
|
|
for item in media_list:
|
|
cursor.execute('''
|
|
INSERT OR IGNORE INTO media (username, tweet_id, url, date, type)
|
|
VALUES (?, ?, ?, ?, ?)
|
|
''', (username, item.get('tweet_id', ''), item['url'],
|
|
item['date'], item.get('type', 'image')))
|
|
|
|
self.conn.commit()
|
|
|
|
def get_account(self, username: str, media_type: str = 'all') -> Optional[Dict]:
|
|
cursor = self.conn.cursor()
|
|
cursor.execute('''
|
|
SELECT * FROM accounts
|
|
WHERE username = ? AND media_type = ?
|
|
''', (username, media_type))
|
|
|
|
row = cursor.fetchone()
|
|
if row:
|
|
return dict(row)
|
|
return None
|
|
|
|
def get_media_list(self, username: str) -> List[Dict]:
|
|
cursor = self.conn.cursor()
|
|
cursor.execute('''
|
|
SELECT tweet_id, url, date, type
|
|
FROM media
|
|
WHERE username = ?
|
|
ORDER BY date DESC
|
|
''', (username,))
|
|
|
|
return [dict(row) for row in cursor.fetchall()]
|
|
|
|
def get_all_accounts(self) -> List[Dict]:
|
|
cursor = self.conn.cursor()
|
|
cursor.execute('SELECT * FROM accounts ORDER BY fetch_timestamp DESC')
|
|
return [dict(row) for row in cursor.fetchall()]
|
|
|
|
def delete_account(self, username: str, media_type: str = None):
|
|
with self._lock:
|
|
cursor = self.conn.cursor()
|
|
|
|
if media_type:
|
|
cursor.execute('DELETE FROM accounts WHERE username = ? AND media_type = ?',
|
|
(username, media_type))
|
|
else:
|
|
cursor.execute('DELETE FROM accounts WHERE username = ?', (username,))
|
|
|
|
self.conn.commit()
|
|
|
|
def is_file_downloaded(self, filepath: str) -> bool:
|
|
cursor = self.conn.cursor()
|
|
filepath_normalized = filepath.lower()
|
|
|
|
cursor.execute('''
|
|
SELECT 1 FROM downloads
|
|
WHERE filepath_normalized = ? AND status = 'downloaded'
|
|
LIMIT 1
|
|
''', (filepath_normalized,))
|
|
|
|
return cursor.fetchone() is not None
|
|
|
|
def is_url_downloaded(self, url: str) -> bool:
|
|
cursor = self.conn.cursor()
|
|
|
|
cursor.execute('''
|
|
SELECT 1 FROM downloads
|
|
WHERE url = ? AND status = 'downloaded'
|
|
LIMIT 1
|
|
''', (url,))
|
|
|
|
return cursor.fetchone() is not None
|
|
|
|
def record_download(self, url: str, filepath: str, username: str,
|
|
tweet_id: str, status: str = 'downloaded', file_size: int = 0):
|
|
with self._lock:
|
|
cursor = self.conn.cursor()
|
|
filepath_normalized = filepath.lower()
|
|
download_date = datetime.now().isoformat()
|
|
|
|
cursor.execute('''
|
|
INSERT OR REPLACE INTO downloads
|
|
(url, filepath, filepath_normalized, username, tweet_id,
|
|
download_date, file_size, status)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
''', (url, filepath, filepath_normalized, username, tweet_id,
|
|
download_date, file_size, status))
|
|
|
|
self.conn.commit()
|
|
|
|
def get_download_stats(self, username: str = None) -> Dict:
|
|
cursor = self.conn.cursor()
|
|
|
|
if username:
|
|
cursor.execute('''
|
|
SELECT
|
|
COUNT(*) as total,
|
|
SUM(CASE WHEN status = 'downloaded' THEN 1 ELSE 0 END) as downloaded,
|
|
SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) as skipped,
|
|
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed,
|
|
SUM(file_size) as total_size
|
|
FROM downloads
|
|
WHERE username = ?
|
|
''', (username,))
|
|
else:
|
|
cursor.execute('''
|
|
SELECT
|
|
COUNT(*) as total,
|
|
SUM(CASE WHEN status = 'downloaded' THEN 1 ELSE 0 END) as downloaded,
|
|
SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) as skipped,
|
|
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed,
|
|
SUM(file_size) as total_size
|
|
FROM downloads
|
|
''')
|
|
|
|
row = cursor.fetchone()
|
|
return dict(row) if row else {}
|
|
|
|
def migrate_from_json(self, json_dir: str) -> Tuple[int, int]:
|
|
if not os.path.exists(json_dir):
|
|
return 0, 0
|
|
|
|
json_files = [f for f in os.listdir(json_dir) if f.endswith('.json')]
|
|
accounts_migrated = 0
|
|
media_migrated = 0
|
|
|
|
for json_file in json_files:
|
|
try:
|
|
json_path = os.path.join(json_dir, json_file)
|
|
|
|
with open(json_path, 'r', encoding='utf-8') as f:
|
|
data = json.load(f)
|
|
|
|
media_type = data.get('media_type', 'all')
|
|
|
|
if media_type not in ['all', 'image', 'video', 'gif']:
|
|
filename_parts = json_file.replace('.json', '').split('_')
|
|
if len(filename_parts) >= 1:
|
|
last_part = filename_parts[-1]
|
|
if last_part in ['all', 'image', 'video', 'gif']:
|
|
media_type = last_part
|
|
else:
|
|
media_type = 'all'
|
|
else:
|
|
media_type = 'all'
|
|
|
|
filename_parts = json_file.replace('.json', '').split('_')
|
|
if len(filename_parts) >= 2:
|
|
second_last = filename_parts[-2]
|
|
if second_last in ['media', 'timeline', 'tweets', 'with']:
|
|
timeline_type = second_last
|
|
else:
|
|
timeline_type = 'media'
|
|
else:
|
|
timeline_type = 'media'
|
|
|
|
account_info = data.get('account_info', {})
|
|
timeline = data.get('timeline', [])
|
|
|
|
username = data.get('username') or account_info.get('name', filename_parts[0] if filename_parts else 'unknown')
|
|
|
|
nick = data.get('nick') or account_info.get('nick') or account_info.get('name', username)
|
|
followers = data.get('followers') or account_info.get('followers_count', 0)
|
|
following = data.get('following') or account_info.get('friends_count', 0)
|
|
posts = data.get('posts') or account_info.get('statuses_count', 0)
|
|
profile_image = data.get('profile_image') or account_info.get('profile_image')
|
|
|
|
self.save_account(
|
|
username=username,
|
|
nick=nick,
|
|
followers=followers,
|
|
following=following,
|
|
posts=posts,
|
|
media_type=media_type,
|
|
profile_image=profile_image,
|
|
fetch_mode=data.get('fetch_mode', 'all'),
|
|
timeline_type=timeline_type
|
|
)
|
|
accounts_migrated += 1
|
|
|
|
media_list = timeline if timeline else data.get('media_list', [])
|
|
if media_list:
|
|
self.save_media_list(username, media_list)
|
|
media_migrated += len(media_list)
|
|
|
|
backup_path = json_path + '.backup'
|
|
if not os.path.exists(backup_path):
|
|
os.rename(json_path, backup_path)
|
|
|
|
except Exception as e:
|
|
print(f"Error migrating {json_file}: {e}")
|
|
continue
|
|
|
|
return accounts_migrated, media_migrated
|
|
|
|
def export_to_json(self, username: str, media_type: str, output_path: str):
|
|
account = self.get_account(username, media_type)
|
|
if not account:
|
|
return False
|
|
|
|
media_list = self.get_media_list(username)
|
|
|
|
export_data = {
|
|
'username': account['username'],
|
|
'nick': account['nick'],
|
|
'followers': account['followers'],
|
|
'following': account['following'],
|
|
'posts': account['posts'],
|
|
'profile_image': account['profile_image'],
|
|
'fetch_timestamp': account['fetch_timestamp'],
|
|
'media_type': account['media_type'],
|
|
'fetch_mode': account['fetch_mode'],
|
|
'media_list': media_list
|
|
}
|
|
|
|
with open(output_path, 'w', encoding='utf-8') as f:
|
|
json.dump(export_data, f, ensure_ascii=False, indent=2)
|
|
|
|
return True
|
|
|
|
def import_from_json(self, json_path: str) -> bool:
|
|
try:
|
|
with open(json_path, 'r', encoding='utf-8') as f:
|
|
data = json.load(f)
|
|
|
|
username = data.get('username')
|
|
if not username:
|
|
return False
|
|
|
|
self.save_account(
|
|
username=username,
|
|
nick=data.get('nick', ''),
|
|
followers=data.get('followers', 0),
|
|
following=data.get('following', 0),
|
|
posts=data.get('posts', 0),
|
|
media_type=data.get('media_type', 'all'),
|
|
profile_image=data.get('profile_image'),
|
|
fetch_mode=data.get('fetch_mode', 'all')
|
|
)
|
|
|
|
media_list = data.get('media_list', [])
|
|
if media_list:
|
|
self.save_media_list(username, media_list)
|
|
|
|
return True
|
|
|
|
except Exception as e:
|
|
print(f"Error importing JSON: {e}")
|
|
return False
|
|
|
|
def close(self):
|
|
if self.conn:
|
|
self.conn.close()
|
|
|
|
def create_group(self, name: str) -> Optional[int]:
|
|
with self._lock:
|
|
try:
|
|
cursor = self.conn.cursor()
|
|
created_at = datetime.now().isoformat()
|
|
cursor.execute('INSERT INTO groups (name, created_at) VALUES (?, ?)', (name, created_at))
|
|
self.conn.commit()
|
|
return cursor.lastrowid
|
|
except sqlite3.IntegrityError:
|
|
return None
|
|
|
|
def get_all_groups(self) -> List[Dict]:
|
|
cursor = self.conn.cursor()
|
|
cursor.execute('SELECT * FROM groups ORDER BY name')
|
|
return [dict(row) for row in cursor.fetchall()]
|
|
|
|
def get_group(self, group_id: int) -> Optional[Dict]:
|
|
cursor = self.conn.cursor()
|
|
cursor.execute('SELECT * FROM groups WHERE id = ?', (group_id,))
|
|
row = cursor.fetchone()
|
|
return dict(row) if row else None
|
|
|
|
def update_group(self, group_id: int, name: str) -> bool:
|
|
with self._lock:
|
|
try:
|
|
cursor = self.conn.cursor()
|
|
cursor.execute('UPDATE groups SET name = ? WHERE id = ?', (name, group_id))
|
|
self.conn.commit()
|
|
return True
|
|
except sqlite3.IntegrityError:
|
|
return False
|
|
|
|
def delete_group(self, group_id: int):
|
|
with self._lock:
|
|
cursor = self.conn.cursor()
|
|
cursor.execute('UPDATE accounts SET group_id = NULL WHERE group_id = ?', (group_id,))
|
|
cursor.execute('DELETE FROM groups WHERE id = ?', (group_id,))
|
|
self.conn.commit()
|
|
|
|
def assign_account_to_group(self, username: str, group_id: Optional[int]):
|
|
with self._lock:
|
|
cursor = self.conn.cursor()
|
|
cursor.execute('UPDATE accounts SET group_id = ? WHERE username = ?', (group_id, username))
|
|
self.conn.commit()
|
|
|
|
def get_accounts_by_group(self, group_id: Optional[int]) -> List[Dict]:
|
|
cursor = self.conn.cursor()
|
|
if group_id is None:
|
|
cursor.execute('SELECT * FROM accounts WHERE group_id IS NULL ORDER BY fetch_timestamp DESC')
|
|
else:
|
|
cursor.execute('SELECT * FROM accounts WHERE group_id = ? ORDER BY fetch_timestamp DESC', (group_id,))
|
|
return [dict(row) for row in cursor.fetchall()]
|