Backups/yt-dlp
1
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2025-10-04 11:26:40 -04:00
Code Issues Projects Releases Wiki Activity

Standardize retry mechanism (#1649)

Browse Source 
* [utils] Create `RetryManager`
* Migrate all retries to use the manager
* [extractor] Add wrapper methods for convenience
* Standardize console messages for retries
* Add `--retry-sleep` for extractors
This commit is contained in:
pukkandan
2022-08-02 01:43:18 +05:30
committed by GitHub
parent bfd973ece3
commit be5c1ae862
15 changed files with 256 additions and 277 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
yt_dlp/extractor/tiktok.py
View File

@@ -630,19 +630,17 @@ class TikTokUserIE(TikTokBaseIE):
'device_id': ''.join(random.choice(string.digits) for _ in range(19)), # Some endpoints don't like randomized device_id, so it isn't directly set in _call_api.
}
max_retries = self.get_param('extractor_retries', 3)
for page in itertools.count(1):
for retries in itertools.count():
for retry in self.RetryManager():
try:
post_list = self._call_api('aweme/post', query, username,
note='Downloading user video list page %d%s' % (page, f' (attempt {retries})' if retries != 0 else ''),
errnote='Unable to download user video list')
post_list = self._call_api(
'aweme/post', query, username, note=f'Downloading user video list page {page}',
errnote='Unable to download user video list')
except ExtractorError as e:
if isinstance(e.cause, json.JSONDecodeError) and e.cause.pos == 0 and retries != max_retries:
self.report_warning('%s. Retrying...' % str(e.cause or e.msg))
if isinstance(e.cause, json.JSONDecodeError) and e.cause.pos == 0:
retry.error = e
continue
raise
break
yield from post_list.get('aweme_list', [])
if not post_list.get('has_more'):
break
@@ -680,19 +678,17 @@ class TikTokBaseListIE(TikTokBaseIE):
'device_id': ''.join(random.choice(string.digits) for i in range(19))
}
max_retries = self.get_param('extractor_retries', 3)
for page in itertools.count(1):
for retries in itertools.count():
for retry in self.RetryManager():
try:
post_list = self._call_api(self._API_ENDPOINT, query, display_id,
note='Downloading video list page %d%s' % (page, f' (attempt {retries})' if retries != 0 else ''),
errnote='Unable to download video list')
post_list = self._call_api(
self._API_ENDPOINT, query, display_id, note=f'Downloading video list page {page}',
errnote='Unable to download video list')
except ExtractorError as e:
if isinstance(e.cause, json.JSONDecodeError) and e.cause.pos == 0 and retries != max_retries:
self.report_warning('%s. Retrying...' % str(e.cause or e.msg))
if isinstance(e.cause, json.JSONDecodeError) and e.cause.pos == 0:
retry.error = e
continue
raise
break
for video in post_list.get('aweme_list', []):
yield {
**self._parse_aweme_video_app(video),