mirror of
https://github.com/Alvin-Zilverstand/Twitter-X-Media-Batch-Downloader.git
synced 2026-11-21 13:53:13 +01:00
.
This commit is contained in:
1 parent
61e50b8121
commit
c894bfd4d2
5 files changed
-2024
No files matched your search
@@ -1,209 +0,0 @@
|
||||
import json
|
||||
import re
|
||||
import os
|
||||
from datetime import datetime
|
||||
from gallery_dl.extractor import twitter
|
||||
|
||||
def get_metadata(username, auth_token, timeline_type="timeline", batch_size=0, page=0, output_file=None):
|
||||
url = f"https://x.com/{username}/{timeline_type}"
|
||||
|
||||
if timeline_type == "media":
|
||||
extractor_class = twitter.TwitterMediaExtractor
|
||||
elif timeline_type == "tweets":
|
||||
extractor_class = twitter.TwitterTweetsExtractor
|
||||
elif timeline_type == "with_replies":
|
||||
extractor_class = twitter.TwitterRepliesExtractor
|
||||
else:
|
||||
extractor_class = twitter.TwitterTimelineExtractor
|
||||
|
||||
match = re.match(extractor_class.pattern, url)
|
||||
if not match:
|
||||
raise ValueError(f"Invalid URL for {timeline_type}: {url}")
|
||||
|
||||
extractor = extractor_class(match)
|
||||
|
||||
config_dict = {
|
||||
"cookies": {
|
||||
"auth_token": auth_token
|
||||
}
|
||||
}
|
||||
|
||||
if batch_size > 0:
|
||||
config_dict["count"] = batch_size
|
||||
|
||||
extractor.config = lambda key, default=None: config_dict.get(key, default)
|
||||
|
||||
extractor.initialize()
|
||||
|
||||
if output_file is None:
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
output_file = os.path.join(current_dir, f"{username}_{timeline_type}.json")
|
||||
|
||||
structured_output = {
|
||||
'account_info': {},
|
||||
'total_urls': 0,
|
||||
'timeline': []
|
||||
}
|
||||
|
||||
if page > 0 and os.path.exists(output_file):
|
||||
try:
|
||||
with open(output_file, 'r', encoding='utf-8') as f:
|
||||
structured_output = json.load(f)
|
||||
print(f"Loaded existing data with {structured_output['total_urls']} items from {output_file}")
|
||||
except json.JSONDecodeError:
|
||||
print(f"Warning: Could not load existing data from {output_file}, starting fresh")
|
||||
|
||||
iterator = iter(extractor)
|
||||
|
||||
if batch_size > 0 and page > 0:
|
||||
items_to_skip = page * batch_size
|
||||
print(f"Skipping {items_to_skip} items for page {page}...")
|
||||
|
||||
if hasattr(extractor, '_cursor') and extractor._cursor:
|
||||
print(f"Using cursor: {extractor._cursor}")
|
||||
else:
|
||||
skipped = 0
|
||||
try:
|
||||
for _ in range(items_to_skip):
|
||||
next(iterator)
|
||||
skipped += 1
|
||||
except StopIteration:
|
||||
print(f"Warning: Could only skip {skipped} items")
|
||||
|
||||
batch_items = []
|
||||
new_timeline_entries = []
|
||||
|
||||
items_to_fetch = batch_size if batch_size > 0 else float('inf')
|
||||
items_fetched = 0
|
||||
|
||||
try:
|
||||
while items_fetched < items_to_fetch:
|
||||
item = next(iterator)
|
||||
batch_items.append(item)
|
||||
items_fetched += 1
|
||||
|
||||
if isinstance(item, tuple) and len(item) >= 3:
|
||||
media_url = item[1]
|
||||
tweet_data = item[2]
|
||||
|
||||
if not structured_output['account_info'] and 'user' in tweet_data:
|
||||
user = tweet_data['user']
|
||||
user_date = user.get('date', '')
|
||||
if isinstance(user_date, datetime):
|
||||
user_date = user_date.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
structured_output['account_info'] = {
|
||||
'name': user.get('name', ''),
|
||||
'nick': user.get('nick', ''),
|
||||
'date': user_date,
|
||||
'followers_count': user.get('followers_count', 0),
|
||||
'friends_count': user.get('friends_count', 0),
|
||||
'profile_image': user.get('profile_image', ''),
|
||||
'statuses_count': user.get('statuses_count', 0)
|
||||
}
|
||||
|
||||
if 'pbs.twimg.com' in media_url or 'video.twimg.com' in media_url:
|
||||
tweet_date = tweet_data.get('date', datetime.now())
|
||||
if isinstance(tweet_date, datetime):
|
||||
tweet_date = tweet_date.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
timeline_entry = {
|
||||
'url': media_url,
|
||||
'date': tweet_date,
|
||||
'tweet_id': tweet_data.get('tweet_id', 0),
|
||||
}
|
||||
|
||||
if 'type' in tweet_data:
|
||||
timeline_entry['type'] = tweet_data['type']
|
||||
|
||||
new_timeline_entries.append(timeline_entry)
|
||||
structured_output['total_urls'] += 1
|
||||
except StopIteration:
|
||||
pass
|
||||
|
||||
structured_output['timeline'].extend(new_timeline_entries)
|
||||
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(structured_output, f, ensure_ascii=False, indent=2)
|
||||
|
||||
if batch_size > 0:
|
||||
print(f"Page {page}: Added {len(new_timeline_entries)} media URLs. Total URLs: {structured_output['total_urls']}")
|
||||
else:
|
||||
print(f"Added {len(new_timeline_entries)} media URLs. Total URLs: {structured_output['total_urls']}")
|
||||
|
||||
cursor_info = None
|
||||
if hasattr(extractor, '_cursor') and extractor._cursor:
|
||||
cursor_info = extractor._cursor
|
||||
print(f"Current cursor: {cursor_info}")
|
||||
|
||||
return {
|
||||
"items": batch_items,
|
||||
"structured_output": structured_output,
|
||||
"new_entries": len(new_timeline_entries),
|
||||
"total_urls": structured_output['total_urls'],
|
||||
"page": page,
|
||||
"batch_size": batch_size,
|
||||
"has_more": batch_size > 0 and items_fetched == batch_size,
|
||||
"cursor": cursor_info
|
||||
}
|
||||
|
||||
def continue_metadata_fetch(username, auth_token, timeline_type="media", batch_size=100, max_pages=None):
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
output_file = os.path.join(current_dir, f"{username}_{timeline_type}.json")
|
||||
|
||||
page = 0
|
||||
total_urls = 0
|
||||
has_more = True
|
||||
|
||||
if os.path.exists(output_file):
|
||||
try:
|
||||
with open(output_file, 'r', encoding='utf-8') as f:
|
||||
existing_data = json.load(f)
|
||||
total_urls = existing_data.get('total_urls', 0)
|
||||
page = total_urls // batch_size if batch_size > 0 else 0
|
||||
print(f"Continuing from page {page} with {total_urls} existing URLs")
|
||||
except json.JSONDecodeError:
|
||||
print(f"Warning: Could not load existing data from {output_file}, starting fresh")
|
||||
|
||||
while has_more and (max_pages is None or page < max_pages):
|
||||
result = get_metadata(
|
||||
username=username,
|
||||
auth_token=auth_token,
|
||||
timeline_type=timeline_type,
|
||||
batch_size=batch_size,
|
||||
page=page,
|
||||
output_file=output_file
|
||||
)
|
||||
|
||||
has_more = result["has_more"]
|
||||
total_urls = result["total_urls"]
|
||||
|
||||
if not has_more:
|
||||
print(f"No more items to fetch. Total URLs: {total_urls}")
|
||||
break
|
||||
|
||||
page += 1
|
||||
|
||||
return total_urls
|
||||
|
||||
# Usage
|
||||
if __name__ == "__main__":
|
||||
username = "takomayuyi"
|
||||
auth_token = ""
|
||||
|
||||
# Example 1: Non-batch mode - fetch all items at once
|
||||
# This will fetch all items without pagination
|
||||
# result = get_metadata(username, auth_token, "media")
|
||||
# print(f"Non-batch mode: Added {result['new_entries']} media URLs")
|
||||
|
||||
# Example 2: Batch mode - fetch items in pages
|
||||
# This will fetch only the specified page with the given batch size
|
||||
result = get_metadata(username, auth_token, "media", batch_size=100, page=0)
|
||||
print(f"Batch mode: Added {result['new_entries']} media URLs (page 0)")
|
||||
|
||||
# To fetch the next page:
|
||||
# result = get_metadata(username, auth_token, "media", batch_size=100, page=1)
|
||||
|
||||
# To automatically fetch all pages in batch mode:
|
||||
# total = continue_metadata_fetch(username, auth_token, "media", batch_size=100)
|
||||
# print(f"Total media URLs fetched across all pages: {total}")
|
||||
@@ -1,161 +0,0 @@
|
||||
import json
|
||||
import re
|
||||
import os
|
||||
from gallery_dl.extractor import twitter
|
||||
|
||||
def get_metadata(username, auth_token, timeline_type="timeline", batch_size=0, page=0, output_file=None):
|
||||
url = f"https://x.com/{username}/{timeline_type}"
|
||||
|
||||
if timeline_type == "media":
|
||||
extractor_class = twitter.TwitterMediaExtractor
|
||||
elif timeline_type == "tweets":
|
||||
extractor_class = twitter.TwitterTweetsExtractor
|
||||
elif timeline_type == "with_replies":
|
||||
extractor_class = twitter.TwitterRepliesExtractor
|
||||
else:
|
||||
extractor_class = twitter.TwitterTimelineExtractor
|
||||
|
||||
match = re.match(extractor_class.pattern, url)
|
||||
if not match:
|
||||
raise ValueError(f"Invalid URL for {timeline_type}: {url}")
|
||||
|
||||
extractor = extractor_class(match)
|
||||
|
||||
config_dict = {
|
||||
"cookies": {
|
||||
"auth_token": auth_token
|
||||
}
|
||||
}
|
||||
|
||||
if batch_size > 0:
|
||||
config_dict["count"] = batch_size
|
||||
|
||||
extractor.config = lambda key, default=None: config_dict.get(key, default)
|
||||
|
||||
extractor.initialize()
|
||||
|
||||
if output_file is None:
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
output_file = os.path.join(current_dir, f"{username}_{timeline_type}.json")
|
||||
|
||||
all_items = []
|
||||
if page > 0 and os.path.exists(output_file):
|
||||
try:
|
||||
with open(output_file, 'r', encoding='utf-8') as f:
|
||||
all_items = json.load(f)
|
||||
print(f"Loaded {len(all_items)} existing items from {output_file}")
|
||||
except json.JSONDecodeError:
|
||||
print(f"Warning: Could not load existing data from {output_file}, starting fresh")
|
||||
|
||||
iterator = iter(extractor)
|
||||
|
||||
if batch_size > 0 and page > 0:
|
||||
items_to_skip = page * batch_size
|
||||
print(f"Skipping {items_to_skip} items for page {page}...")
|
||||
|
||||
if hasattr(extractor, '_cursor') and extractor._cursor:
|
||||
print(f"Using cursor: {extractor._cursor}")
|
||||
else:
|
||||
skipped = 0
|
||||
try:
|
||||
for _ in range(items_to_skip):
|
||||
next(iterator)
|
||||
skipped += 1
|
||||
except StopIteration:
|
||||
print(f"Warning: Could only skip {skipped} items")
|
||||
|
||||
batch_items = []
|
||||
if batch_size > 0:
|
||||
try:
|
||||
for _ in range(batch_size):
|
||||
item = next(iterator)
|
||||
batch_items.append(item)
|
||||
except StopIteration:
|
||||
pass
|
||||
else:
|
||||
batch_items = list(iterator)
|
||||
|
||||
all_items.extend(batch_items)
|
||||
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(all_items, f, indent=2, default=str)
|
||||
|
||||
if batch_size > 0:
|
||||
print(f"Page {page}: Retrieved {len(batch_items)} items. Total items: {len(all_items)}")
|
||||
else:
|
||||
print(f"Retrieved {len(batch_items)} items total")
|
||||
|
||||
cursor_info = None
|
||||
if hasattr(extractor, '_cursor') and extractor._cursor:
|
||||
cursor_info = extractor._cursor
|
||||
print(f"Current cursor: {cursor_info}")
|
||||
|
||||
return {
|
||||
"items": batch_items,
|
||||
"total_items": len(all_items),
|
||||
"page": page,
|
||||
"batch_size": batch_size,
|
||||
"has_more": batch_size > 0 and len(batch_items) == batch_size,
|
||||
"cursor": cursor_info
|
||||
}
|
||||
|
||||
def continue_metadata_fetch(username, auth_token, timeline_type="media", batch_size=100, max_pages=None):
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
output_file = os.path.join(current_dir, f"{username}_{timeline_type}.json")
|
||||
|
||||
page = 0
|
||||
total_items = 0
|
||||
has_more = True
|
||||
|
||||
if os.path.exists(output_file):
|
||||
try:
|
||||
with open(output_file, 'r', encoding='utf-8') as f:
|
||||
existing_data = json.load(f)
|
||||
total_items = len(existing_data)
|
||||
page = total_items // batch_size
|
||||
print(f"Continuing from page {page} with {total_items} existing items")
|
||||
except json.JSONDecodeError:
|
||||
print(f"Warning: Could not load existing data from {output_file}, starting fresh")
|
||||
|
||||
while has_more and (max_pages is None or page < max_pages):
|
||||
result = get_metadata(
|
||||
username=username,
|
||||
auth_token=auth_token,
|
||||
timeline_type=timeline_type,
|
||||
batch_size=batch_size,
|
||||
page=page,
|
||||
output_file=output_file
|
||||
)
|
||||
|
||||
has_more = result["has_more"]
|
||||
total_items = result["total_items"]
|
||||
|
||||
if not has_more:
|
||||
print(f"No more items to fetch. Total items: {total_items}")
|
||||
break
|
||||
|
||||
page += 1
|
||||
|
||||
return total_items
|
||||
|
||||
# Usage
|
||||
if __name__ == "__main__":
|
||||
username = "takomayuyi"
|
||||
auth_token = ""
|
||||
|
||||
# Example 1: Non-batch mode - fetch all items at once
|
||||
# This will fetch all items without pagination
|
||||
# result = get_metadata(username, auth_token, "media")
|
||||
# print(f"Non-batch mode: Retrieved {len(result['items'])} items")
|
||||
|
||||
# Example 2: Batch mode - fetch items in pages
|
||||
# This will fetch only the specified page with the given batch size
|
||||
result = get_metadata(username, auth_token, "media", batch_size=100, page=0)
|
||||
print(f"Batch mode: Retrieved {len(result['items'])} items (page 0)")
|
||||
|
||||
# To fetch the next page:
|
||||
# result = get_metadata(username, auth_token, "media", batch_size=100, page=1)
|
||||
|
||||
# To automatically fetch all pages in batch mode:
|
||||
# total = continue_metadata_fetch(username, auth_token, "media", batch_size=100)
|
||||
# print(f"Total items fetched across all pages: {total}")
|
||||
@@ -1,11 +0,0 @@
|
||||
<?xml version="1.0" ?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1"
|
||||
id="Layer_1" width="512px" height="512px" viewBox="0 0 24 24" style="enable-background:new 0 0 24 24;"
|
||||
xml:space="preserve">
|
||||
<g fill="#1da0f1">
|
||||
<polygon
|
||||
points="12.153992,10.729553 8.088684,5.041199 5.92041,5.041199 10.956299,12.087097 11.59021,12.97345 15.900635,19.009583 18.068909,19.009583 12.785217,11.615906 " />
|
||||
<path
|
||||
d="M21.15979,1H2.84021C1.823853,1,1,1.823853,1,2.84021v18.31958C1,22.176147,1.823853,23,2.84021,23h18.31958 C22.176147,23,23,22.176147,23,21.15979V2.84021C23,1.823853,22.176147,1,21.15979,1z M15.235352,20l-4.362549-6.213013 L5.411438,20H4l6.246887-7.104675L4,4h4.764648l4.130127,5.881958L18.06958,4h1.411377l-5.95697,6.775635L20,20H15.235352z" />
|
||||
</g>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 876 B |
File diff suppressed because it is too large.
Load diff
-186
@@ -1,186 +0,0 @@
|
||||
import re
|
||||
import json
|
||||
from datetime import datetime
|
||||
from gallery_dl.extractor import twitter
|
||||
|
||||
def get_metadata(username, auth_token, timeline_type="timeline", batch_size=0, page=0, media_type="all"):
|
||||
url = f"https://x.com/{username}/{timeline_type}"
|
||||
|
||||
if timeline_type == "media":
|
||||
extractor_class = twitter.TwitterMediaExtractor
|
||||
elif timeline_type == "tweets":
|
||||
extractor_class = twitter.TwitterTweetsExtractor
|
||||
elif timeline_type == "with_replies":
|
||||
extractor_class = twitter.TwitterRepliesExtractor
|
||||
else:
|
||||
extractor_class = twitter.TwitterTimelineExtractor
|
||||
|
||||
match = re.match(extractor_class.pattern, url)
|
||||
if not match:
|
||||
raise ValueError(f"Invalid URL for {timeline_type}: {url}")
|
||||
|
||||
extractor = extractor_class(match)
|
||||
|
||||
config_dict = {
|
||||
"cookies": {
|
||||
"auth_token": auth_token
|
||||
}
|
||||
}
|
||||
|
||||
if batch_size > 0:
|
||||
config_dict["count"] = batch_size
|
||||
|
||||
extractor.config = lambda key, default=None: config_dict.get(key, default)
|
||||
|
||||
try:
|
||||
extractor.initialize()
|
||||
|
||||
api = twitter.TwitterAPI(extractor)
|
||||
try:
|
||||
if username.startswith("id:"):
|
||||
user = api.user_by_rest_id(username[3:])
|
||||
else:
|
||||
user = api.user_by_screen_name(username)
|
||||
|
||||
if "legacy" in user and user["legacy"].get("withheld_scope"):
|
||||
raise ValueError("withheld")
|
||||
|
||||
except Exception as e:
|
||||
error_msg = str(e).lower()
|
||||
if "withheld" in error_msg or (hasattr(e, "response") and "withheld" in str(e.response.text).lower()):
|
||||
raise ValueError("withheld")
|
||||
raise
|
||||
|
||||
structured_output = {
|
||||
'account_info': {},
|
||||
'total_urls': 0,
|
||||
'timeline': []
|
||||
}
|
||||
|
||||
iterator = iter(extractor)
|
||||
|
||||
if batch_size > 0 and page > 0:
|
||||
items_to_skip = page * batch_size
|
||||
|
||||
if hasattr(extractor, '_cursor') and extractor._cursor:
|
||||
pass
|
||||
else:
|
||||
skipped = 0
|
||||
try:
|
||||
for _ in range(items_to_skip):
|
||||
next(iterator)
|
||||
skipped += 1
|
||||
except StopIteration:
|
||||
pass
|
||||
|
||||
new_timeline_entries = []
|
||||
|
||||
items_to_fetch = batch_size if batch_size > 0 else float('inf')
|
||||
items_fetched = 0
|
||||
|
||||
try:
|
||||
while items_fetched < items_to_fetch:
|
||||
item = next(iterator)
|
||||
items_fetched += 1
|
||||
|
||||
if isinstance(item, tuple) and len(item) >= 3:
|
||||
media_url = item[1]
|
||||
tweet_data = item[2]
|
||||
|
||||
if not structured_output['account_info'] and 'user' in tweet_data:
|
||||
user = tweet_data['user']
|
||||
user_date = user.get('date', '')
|
||||
if isinstance(user_date, datetime):
|
||||
user_date = user_date.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
structured_output['account_info'] = {
|
||||
'name': user.get('name', ''),
|
||||
'nick': user.get('nick', ''),
|
||||
'date': user_date,
|
||||
'followers_count': user.get('followers_count', 0),
|
||||
'friends_count': user.get('friends_count', 0),
|
||||
'profile_image': user.get('profile_image', ''),
|
||||
'statuses_count': user.get('statuses_count', 0)
|
||||
}
|
||||
|
||||
if 'pbs.twimg.com' in media_url or 'video.twimg.com' in media_url:
|
||||
tweet_date = tweet_data.get('date', datetime.now())
|
||||
if isinstance(tweet_date, datetime):
|
||||
tweet_date = tweet_date.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
timeline_entry = {
|
||||
'url': media_url,
|
||||
'date': tweet_date,
|
||||
'tweet_id': tweet_data.get('tweet_id', 0),
|
||||
}
|
||||
|
||||
if 'type' in tweet_data:
|
||||
timeline_entry['type'] = tweet_data['type']
|
||||
|
||||
if media_type == 'all' or (
|
||||
(media_type == 'image' and 'pbs.twimg.com' in media_url and tweet_data.get('type') == 'photo') or
|
||||
(media_type == 'video' and 'video.twimg.com' in media_url and tweet_data.get('type') == 'video') or
|
||||
(media_type == 'gif' and 'video.twimg.com' in media_url and tweet_data.get('type') == 'animated_gif')
|
||||
):
|
||||
new_timeline_entries.append(timeline_entry)
|
||||
structured_output['total_urls'] += 1
|
||||
except StopIteration:
|
||||
pass
|
||||
|
||||
structured_output['timeline'].extend(new_timeline_entries)
|
||||
|
||||
cursor_info = None
|
||||
if hasattr(extractor, '_cursor') and extractor._cursor:
|
||||
cursor_info = extractor._cursor
|
||||
|
||||
structured_output['metadata'] = {
|
||||
"new_entries": len(new_timeline_entries),
|
||||
"page": page,
|
||||
"batch_size": batch_size,
|
||||
"has_more": batch_size > 0 and items_fetched == batch_size,
|
||||
"cursor": cursor_info
|
||||
}
|
||||
|
||||
if not structured_output['account_info']:
|
||||
raise ValueError("Failed to fetch account information. Please check the username and auth token.")
|
||||
|
||||
return structured_output
|
||||
|
||||
except Exception as e:
|
||||
error_msg = str(e).lower()
|
||||
if "withheld" in error_msg or e.__class__.__name__ == "ValueError" and str(e) == "withheld":
|
||||
return {"error": "To download withheld accounts, use this userscript version: https://www.patreon.com/exyezed"}
|
||||
else:
|
||||
error_str = str(e)
|
||||
if error_str == "None":
|
||||
return {"error": "Failed to authenticate. Please verify your auth token is valid and not expired."}
|
||||
else:
|
||||
return {"error": error_str}
|
||||
|
||||
def main():
|
||||
username = ""
|
||||
auth_token = ""
|
||||
timeline_type = "media"
|
||||
batch_size = 100
|
||||
page = 0
|
||||
media_type = "all"
|
||||
|
||||
try:
|
||||
data = get_metadata(
|
||||
username=username,
|
||||
auth_token=auth_token,
|
||||
timeline_type=timeline_type,
|
||||
batch_size=batch_size,
|
||||
page=page,
|
||||
media_type=media_type
|
||||
)
|
||||
print(json.dumps(data, ensure_ascii=False, indent=2))
|
||||
except Exception as e:
|
||||
error_str = str(e)
|
||||
if error_str == "None":
|
||||
print(json.dumps({"error": "Failed to authenticate. Please verify your auth token is valid and not expired."}, ensure_ascii=False))
|
||||
else:
|
||||
print(json.dumps({"error": error_str}, ensure_ascii=False))
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user