From 41f5fd818b75e4a119854065ed3d05daaef53c93 Mon Sep 17 00:00:00 2001 From: Niyazi Suleymanov Date: Sat, 16 Jul 2022 22:52:19 +0400 Subject: [PATCH 1/6] entry no longer exists --- youtube_dl/extractor/newgrounds.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py index 82e7cf522..f65843358 100644 --- a/youtube_dl/extractor/newgrounds.py +++ b/youtube_dl/extractor/newgrounds.py @@ -26,17 +26,6 @@ class NewgroundsIE(InfoExtractor): 'upload_date': '20130911', 'duration': 143, }, - }, { - 'url': 'https://www.newgrounds.com/portal/view/673111', - 'md5': '3394735822aab2478c31b1004fe5e5bc', - 'info_dict': { - 'id': '673111', - 'ext': 'mp4', - 'title': 'Dancin', - 'uploader': 'Squirrelman82', - 'timestamp': 1460256780, - 'upload_date': '20160410', - }, }, { # source format unavailable, additional mp4 formats 'url': 'http://www.newgrounds.com/portal/view/689400', From 9036cf5132215645248ce804211e43974b7e4952 Mon Sep 17 00:00:00 2001 From: Niyazi Suleymanov Date: Sat, 16 Jul 2022 23:11:50 +0400 Subject: [PATCH 2/6] add new test --- youtube_dl/extractor/newgrounds.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py index f65843358..3f4a5d11f 100644 --- a/youtube_dl/extractor/newgrounds.py +++ b/youtube_dl/extractor/newgrounds.py @@ -26,6 +26,17 @@ class NewgroundsIE(InfoExtractor): 'upload_date': '20130911', 'duration': 143, }, + }, { + 'url': 'https://www.newgrounds.com/portal/view/850292', + 'md5': 'bb7cacf45e1b4d648e2dac2d79284d67', + 'info_dict': { + 'id': '850292', + 'ext': 'mp4', + 'title': 'Timeless (2021)', + 'uploader': 'Kevuhn', + 'timestamp': 1657896960, + 'upload_date': '20220715', + }, }, { # source format unavailable, additional mp4 formats 'url': 'http://www.newgrounds.com/portal/view/689400', @@ -44,7 +55,7 @@ class NewgroundsIE(InfoExtractor): def _real_extract(self, url): media_id = self._match_id(url) - + webpage = self._download_webpage(url, media_id) title = self._html_search_regex( @@ -80,7 +91,6 @@ class NewgroundsIE(InfoExtractor): (r'(?s)]*>(.+?).*?\s*Author\s*', r'(?:Author|Writer)\s*]+>([^<]+)'), webpage, 'uploader', fatal=False) - timestamp = unified_timestamp(self._html_search_regex( (r'
\s*Uploaded\s*
\s*
([^<]+
\s*
[^<]+)', r'
\s*Uploaded\s*
\s*
([^<]+)'), webpage, 'timestamp', From 45ae186416881733ffe8d742107e748e06699948 Mon Sep 17 00:00:00 2001 From: Niyazi Suleymanov Date: Sun, 17 Jul 2022 01:20:01 +0400 Subject: [PATCH 3/6] change incorrect uploader --- youtube_dl/extractor/newgrounds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py index 3f4a5d11f..9cb794ec8 100644 --- a/youtube_dl/extractor/newgrounds.py +++ b/youtube_dl/extractor/newgrounds.py @@ -44,7 +44,7 @@ class NewgroundsIE(InfoExtractor): 'id': '689400', 'ext': 'mp4', 'title': 'ZTV News Episode 8', - 'uploader': 'BennettTheSage', + 'uploader': 'ZONE-SAMA', 'timestamp': 1487965140, 'upload_date': '20170224', }, From f6c4b9da47423bbecc0ef0f60107ffac9e0fbb09 Mon Sep 17 00:00:00 2001 From: Niyazi Suleymanov Date: Sun, 17 Jul 2022 01:27:56 +0400 Subject: [PATCH 4/6] fix #30575 --- youtube_dl/extractor/newgrounds.py | 77 +++++++++++++++++------------- 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py index 9cb794ec8..d6ec468bc 100644 --- a/youtube_dl/extractor/newgrounds.py +++ b/youtube_dl/extractor/newgrounds.py @@ -5,7 +5,6 @@ import re from .common import InfoExtractor from ..utils import ( extract_attributes, - int_or_none, parse_duration, parse_filesize, unified_timestamp, @@ -55,42 +54,50 @@ class NewgroundsIE(InfoExtractor): def _real_extract(self, url): media_id = self._match_id(url) - + webpage = self._download_webpage(url, media_id) + + paths = url.split('/') + if paths[-3] == 'audio': + isAudio = True + else: + isAudio = False + + if isAudio: + media_url = self._parse_json(self._search_regex( + r'"url"\s*:\s*("[^"]+"),', webpage, ''), media_id) + + uploader = self._html_search_regex( + (r'(?s)]*>(.+?).*?\s*Author\s*', + r'(?:Author|Writer)\s*]+>([^<]+)'), webpage, 'uploader', + fatal=False) + + formats = [{ + 'url': media_url, + 'format_id': 'source', + 'quality': 1, + }] + + else: + media_url = f'https://www.newgrounds.com/portal/video/{media_id}' + media = self._download_json(media_url, media_id, headers={'X-Requested-With': 'XMLHttpRequest'}) + + uploader = media['author'] + + sources = media['sources'] + + formats = [] + for source in sources: + for i in range(len(sources[source])): + formats.append({ + 'url': sources[source][i]['src'], + 'format_id': source, + 'height': int(source[:-2]) # 1080p -> 1080 + }) + title = self._html_search_regex( r'([^>]+)', webpage, 'title') - - media_url = self._parse_json(self._search_regex( - r'"url"\s*:\s*("[^"]+"),', webpage, ''), media_id) - - formats = [{ - 'url': media_url, - 'format_id': 'source', - 'quality': 1, - }] - - max_resolution = int_or_none(self._search_regex( - r'max_resolution["\']\s*:\s*(\d+)', webpage, 'max resolution', - default=None)) - if max_resolution: - url_base = media_url.rpartition('.')[0] - for resolution in (360, 720, 1080): - if resolution > max_resolution: - break - formats.append({ - 'url': '%s.%dp.mp4' % (url_base, resolution), - 'format_id': '%dp' % resolution, - 'height': resolution, - }) - - self._check_formats(formats, media_id) - self._sort_formats(formats) - - uploader = self._html_search_regex( - (r'(?s)]*>(.+?).*?\s*Author\s*', - r'(?:Author|Writer)\s*]+>([^<]+)'), webpage, 'uploader', - fatal=False) timestamp = unified_timestamp(self._html_search_regex( (r'
\s*Uploaded\s*
\s*
([^<]+
\s*
[^<]+)', r'
\s*Uploaded\s*
\s*
([^<]+)'), webpage, 'timestamp', @@ -98,7 +105,6 @@ class NewgroundsIE(InfoExtractor): duration = parse_duration(self._search_regex( r'(?s)
\s*Song\s*
\s*
.+?
\s*
([^<]+)', webpage, 'duration', default=None)) - filesize_approx = parse_filesize(self._html_search_regex( r'(?s)
\s*Song\s*
\s*
(.+?)
', webpage, 'filesize', default=None)) @@ -108,6 +114,9 @@ class NewgroundsIE(InfoExtractor): if '
Song' in webpage: formats[0]['vcodec'] = 'none' + self._check_formats(formats, media_id) + self._sort_formats(formats) + return { 'id': media_id, 'title': title, From c848538aff38423d26cff9163aa984156370eea0 Mon Sep 17 00:00:00 2001 From: Niyazi Suleymanov Date: Sun, 17 Jul 2022 01:51:53 +0400 Subject: [PATCH 5/6] follow coding conventions --- youtube_dl/extractor/newgrounds.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py index d6ec468bc..da06316aa 100644 --- a/youtube_dl/extractor/newgrounds.py +++ b/youtube_dl/extractor/newgrounds.py @@ -56,13 +56,13 @@ class NewgroundsIE(InfoExtractor): media_id = self._match_id(url) webpage = self._download_webpage(url, media_id) - + paths = url.split('/') if paths[-3] == 'audio': isAudio = True else: - isAudio = False - + isAudio = False + if isAudio: media_url = self._parse_json(self._search_regex( r'"url"\s*:\s*("[^"]+"),', webpage, ''), media_id) @@ -71,7 +71,7 @@ class NewgroundsIE(InfoExtractor): (r'(?s)]*>(.+?).*?\s*Author\s*', r'(?:Author|Writer)\s*]+>([^<]+)'), webpage, 'uploader', fatal=False) - + formats = [{ 'url': media_url, 'format_id': 'source', @@ -92,10 +92,9 @@ class NewgroundsIE(InfoExtractor): formats.append({ 'url': sources[source][i]['src'], 'format_id': source, - 'height': int(source[:-2]) # 1080p -> 1080 + 'height': int(source[:-2]) # 1080p -> 1080 }) - title = self._html_search_regex( r'([^>]+)', webpage, 'title') timestamp = unified_timestamp(self._html_search_regex( @@ -116,7 +115,7 @@ class NewgroundsIE(InfoExtractor): self._check_formats(formats, media_id) self._sort_formats(formats) - + return { 'id': media_id, 'title': title, From 3091daa8345566b6b7b26d9b505e6fde01059333 Mon Sep 17 00:00:00 2001 From: dirkf Date: Thu, 21 Jul 2022 21:18:56 +0100 Subject: [PATCH 6/6] Update youtube_dl/extractor/newgrounds.py - remove f-string --- youtube_dl/extractor/newgrounds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py index da06316aa..46a6b0225 100644 --- a/youtube_dl/extractor/newgrounds.py +++ b/youtube_dl/extractor/newgrounds.py @@ -79,7 +79,7 @@ class NewgroundsIE(InfoExtractor): }] else: - media_url = f'https://www.newgrounds.com/portal/video/{media_id}' + media_url = 'https://www.newgrounds.com/portal/video/' + media_id media = self._download_json(media_url, media_id, headers={'X-Requested-With': 'XMLHttpRequest'}) uploader = media['author']