Files
Twitter-X-Media-Batch-Downl…/core/databaseManager.py
T
2025-11-18 13:38:16 +07:00

421 lines
16 KiB
Python

import sqlite3
import os
import json
import threading
from datetime import datetime
from pathlib import Path
from typing import List, Dict, Optional, Tuple
class DatabaseManager:
def __init__(self, db_path: str):
self.db_path = db_path
self.conn = None
self._lock = threading.Lock()
self.init_database()
def init_database(self):
self.conn = sqlite3.connect(self.db_path, check_same_thread=False)
self.conn.row_factory = sqlite3.Row
self.conn.execute('PRAGMA foreign_keys = ON')
cursor = self.conn.cursor()
cursor.execute('''
CREATE TABLE IF NOT EXISTS groups (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT UNIQUE NOT NULL,
created_at TEXT
)
''')
cursor.execute('''
CREATE TABLE IF NOT EXISTS accounts (
username TEXT PRIMARY KEY COLLATE NOCASE,
nick TEXT,
followers INTEGER,
following INTEGER,
posts INTEGER,
profile_image TEXT,
fetch_timestamp TEXT,
media_type TEXT,
fetch_mode TEXT,
timeline_type TEXT DEFAULT 'media',
group_id INTEGER,
FOREIGN KEY (group_id) REFERENCES groups(id) ON DELETE SET NULL
)
''')
try:
cursor.execute("SELECT group_id FROM accounts LIMIT 1")
except sqlite3.OperationalError:
cursor.execute("ALTER TABLE accounts ADD COLUMN group_id INTEGER")
self.conn.commit()
cursor.execute('''
CREATE TABLE IF NOT EXISTS media (
id INTEGER PRIMARY KEY AUTOINCREMENT,
username TEXT COLLATE NOCASE,
tweet_id TEXT,
url TEXT,
date TEXT,
type TEXT,
FOREIGN KEY (username) REFERENCES accounts(username) ON DELETE CASCADE,
UNIQUE(username, url)
)
''')
cursor.execute('''
CREATE TABLE IF NOT EXISTS downloads (
id INTEGER PRIMARY KEY AUTOINCREMENT,
url TEXT,
filepath TEXT,
filepath_normalized TEXT COLLATE NOCASE,
username TEXT COLLATE NOCASE,
tweet_id TEXT,
download_date TEXT,
file_size INTEGER,
status TEXT,
UNIQUE(url, filepath_normalized)
)
''')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_filepath_normalized ON downloads(filepath_normalized)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_url ON downloads(url)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_username ON downloads(username)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_media_username ON media(username)')
self.conn.commit()
def save_account(self, username: str, nick: str, followers: int, following: int,
posts: int, media_type: str, profile_image: str = None,
fetch_mode: str = 'all', timeline_type: str = 'media',
fetch_timestamp: str = None, group_id: int = None):
with self._lock:
cursor = self.conn.cursor()
if fetch_timestamp is None:
fetch_timestamp = datetime.now().isoformat()
cursor.execute('''
INSERT OR REPLACE INTO accounts
(username, nick, followers, following, posts, profile_image,
fetch_timestamp, media_type, fetch_mode, timeline_type, group_id)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
''', (username, nick, followers, following, posts, profile_image,
fetch_timestamp, media_type, fetch_mode, timeline_type, group_id))
self.conn.commit()
def save_media_list(self, username: str, media_list: List[Dict]):
with self._lock:
cursor = self.conn.cursor()
cursor.execute('DELETE FROM media WHERE username = ?', (username,))
for item in media_list:
cursor.execute('''
INSERT OR IGNORE INTO media (username, tweet_id, url, date, type)
VALUES (?, ?, ?, ?, ?)
''', (username, item.get('tweet_id', ''), item['url'],
item['date'], item.get('type', 'image')))
self.conn.commit()
def get_account(self, username: str, media_type: str = 'all') -> Optional[Dict]:
cursor = self.conn.cursor()
cursor.execute('''
SELECT * FROM accounts
WHERE username = ? AND media_type = ?
''', (username, media_type))
row = cursor.fetchone()
if row:
return dict(row)
return None
def get_media_list(self, username: str) -> List[Dict]:
cursor = self.conn.cursor()
cursor.execute('''
SELECT tweet_id, url, date, type
FROM media
WHERE username = ?
ORDER BY date DESC
''', (username,))
return [dict(row) for row in cursor.fetchall()]
def get_all_accounts(self) -> List[Dict]:
cursor = self.conn.cursor()
cursor.execute('SELECT * FROM accounts ORDER BY fetch_timestamp DESC')
return [dict(row) for row in cursor.fetchall()]
def delete_account(self, username: str, media_type: str = None):
with self._lock:
cursor = self.conn.cursor()
if media_type:
cursor.execute('DELETE FROM accounts WHERE username = ? AND media_type = ?',
(username, media_type))
else:
cursor.execute('DELETE FROM accounts WHERE username = ?', (username,))
self.conn.commit()
def is_file_downloaded(self, filepath: str) -> bool:
cursor = self.conn.cursor()
filepath_normalized = filepath.lower()
cursor.execute('''
SELECT 1 FROM downloads
WHERE filepath_normalized = ? AND status = 'downloaded'
LIMIT 1
''', (filepath_normalized,))
return cursor.fetchone() is not None
def is_url_downloaded(self, url: str) -> bool:
cursor = self.conn.cursor()
cursor.execute('''
SELECT 1 FROM downloads
WHERE url = ? AND status = 'downloaded'
LIMIT 1
''', (url,))
return cursor.fetchone() is not None
def record_download(self, url: str, filepath: str, username: str,
tweet_id: str, status: str = 'downloaded', file_size: int = 0):
with self._lock:
cursor = self.conn.cursor()
filepath_normalized = filepath.lower()
download_date = datetime.now().isoformat()
cursor.execute('''
INSERT OR REPLACE INTO downloads
(url, filepath, filepath_normalized, username, tweet_id,
download_date, file_size, status)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
''', (url, filepath, filepath_normalized, username, tweet_id,
download_date, file_size, status))
self.conn.commit()
def get_download_stats(self, username: str = None) -> Dict:
cursor = self.conn.cursor()
if username:
cursor.execute('''
SELECT
COUNT(*) as total,
SUM(CASE WHEN status = 'downloaded' THEN 1 ELSE 0 END) as downloaded,
SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) as skipped,
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed,
SUM(file_size) as total_size
FROM downloads
WHERE username = ?
''', (username,))
else:
cursor.execute('''
SELECT
COUNT(*) as total,
SUM(CASE WHEN status = 'downloaded' THEN 1 ELSE 0 END) as downloaded,
SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) as skipped,
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed,
SUM(file_size) as total_size
FROM downloads
''')
row = cursor.fetchone()
return dict(row) if row else {}
def migrate_from_json(self, json_dir: str) -> Tuple[int, int]:
if not os.path.exists(json_dir):
return 0, 0
json_files = [f for f in os.listdir(json_dir) if f.endswith('.json')]
accounts_migrated = 0
media_migrated = 0
for json_file in json_files:
try:
json_path = os.path.join(json_dir, json_file)
with open(json_path, 'r', encoding='utf-8') as f:
data = json.load(f)
media_type = data.get('media_type', 'all')
if media_type not in ['all', 'image', 'video', 'gif']:
filename_parts = json_file.replace('.json', '').split('_')
if len(filename_parts) >= 1:
last_part = filename_parts[-1]
if last_part in ['all', 'image', 'video', 'gif']:
media_type = last_part
else:
media_type = 'all'
else:
media_type = 'all'
filename_parts = json_file.replace('.json', '').split('_')
if len(filename_parts) >= 2:
second_last = filename_parts[-2]
if second_last in ['media', 'timeline', 'tweets', 'with']:
timeline_type = second_last
else:
timeline_type = 'media'
else:
timeline_type = 'media'
account_info = data.get('account_info', {})
timeline = data.get('timeline', [])
username = data.get('username') or account_info.get('name', filename_parts[0] if filename_parts else 'unknown')
nick = data.get('nick') or account_info.get('nick') or account_info.get('name', username)
followers = data.get('followers') or account_info.get('followers_count', 0)
following = data.get('following') or account_info.get('friends_count', 0)
posts = data.get('posts') or account_info.get('statuses_count', 0)
profile_image = data.get('profile_image') or account_info.get('profile_image')
self.save_account(
username=username,
nick=nick,
followers=followers,
following=following,
posts=posts,
media_type=media_type,
profile_image=profile_image,
fetch_mode=data.get('fetch_mode', 'all'),
timeline_type=timeline_type
)
accounts_migrated += 1
media_list = timeline if timeline else data.get('media_list', [])
if media_list:
self.save_media_list(username, media_list)
media_migrated += len(media_list)
backup_path = json_path + '.backup'
if not os.path.exists(backup_path):
os.rename(json_path, backup_path)
except Exception as e:
print(f"Error migrating {json_file}: {e}")
continue
return accounts_migrated, media_migrated
def export_to_json(self, username: str, media_type: str, output_path: str):
account = self.get_account(username, media_type)
if not account:
return False
media_list = self.get_media_list(username)
export_data = {
'username': account['username'],
'nick': account['nick'],
'followers': account['followers'],
'following': account['following'],
'posts': account['posts'],
'profile_image': account['profile_image'],
'fetch_timestamp': account['fetch_timestamp'],
'media_type': account['media_type'],
'fetch_mode': account['fetch_mode'],
'media_list': media_list
}
with open(output_path, 'w', encoding='utf-8') as f:
json.dump(export_data, f, ensure_ascii=False, indent=2)
return True
def import_from_json(self, json_path: str) -> bool:
try:
with open(json_path, 'r', encoding='utf-8') as f:
data = json.load(f)
username = data.get('username')
if not username:
return False
self.save_account(
username=username,
nick=data.get('nick', ''),
followers=data.get('followers', 0),
following=data.get('following', 0),
posts=data.get('posts', 0),
media_type=data.get('media_type', 'all'),
profile_image=data.get('profile_image'),
fetch_mode=data.get('fetch_mode', 'all')
)
media_list = data.get('media_list', [])
if media_list:
self.save_media_list(username, media_list)
return True
except Exception as e:
print(f"Error importing JSON: {e}")
return False
def close(self):
if self.conn:
self.conn.close()
def create_group(self, name: str) -> Optional[int]:
with self._lock:
try:
cursor = self.conn.cursor()
created_at = datetime.now().isoformat()
cursor.execute('INSERT INTO groups (name, created_at) VALUES (?, ?)', (name, created_at))
self.conn.commit()
return cursor.lastrowid
except sqlite3.IntegrityError:
return None
def get_all_groups(self) -> List[Dict]:
cursor = self.conn.cursor()
cursor.execute('SELECT * FROM groups ORDER BY name')
return [dict(row) for row in cursor.fetchall()]
def get_group(self, group_id: int) -> Optional[Dict]:
cursor = self.conn.cursor()
cursor.execute('SELECT * FROM groups WHERE id = ?', (group_id,))
row = cursor.fetchone()
return dict(row) if row else None
def update_group(self, group_id: int, name: str) -> bool:
with self._lock:
try:
cursor = self.conn.cursor()
cursor.execute('UPDATE groups SET name = ? WHERE id = ?', (name, group_id))
self.conn.commit()
return True
except sqlite3.IntegrityError:
return False
def delete_group(self, group_id: int):
with self._lock:
cursor = self.conn.cursor()
cursor.execute('UPDATE accounts SET group_id = NULL WHERE group_id = ?', (group_id,))
cursor.execute('DELETE FROM groups WHERE id = ?', (group_id,))
self.conn.commit()
def assign_account_to_group(self, username: str, group_id: Optional[int]):
with self._lock:
cursor = self.conn.cursor()
cursor.execute('UPDATE accounts SET group_id = ? WHERE username = ?', (group_id, username))
self.conn.commit()
def get_accounts_by_group(self, group_id: Optional[int]) -> List[Dict]:
cursor = self.conn.cursor()
if group_id is None:
cursor.execute('SELECT * FROM accounts WHERE group_id IS NULL ORDER BY fetch_timestamp DESC')
else:
cursor.execute('SELECT * FROM accounts WHERE group_id = ? ORDER BY fetch_timestamp DESC', (group_id,))
return [dict(row) for row in cursor.fetchall()]