From 2b8eac2d3ab49111271a7b38efad145cdc2a4521 Mon Sep 17 00:00:00 2001 From: alyssadev Date: Tue, 29 Mar 2022 18:17:31 +1000 Subject: [PATCH 1/4] [nbl] Add new extractor --- youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/nbl.py | 69 ++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 youtube_dl/extractor/nbl.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 535080d0a..e48f67424 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -749,6 +749,7 @@ from .nbc import ( NBCSportsStreamIE, NBCSportsVPlayerIE, ) +from .nbl import NBLIE from .ndr import ( NDRIE, NJoyIE, diff --git a/youtube_dl/extractor/nbl.py b/youtube_dl/extractor/nbl.py new file mode 100644 index 000000000..0c3c1e159 --- /dev/null +++ b/youtube_dl/extractor/nbl.py @@ -0,0 +1,69 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor +from ..utils import unified_strdate + + +class NBLIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?nbl\.com\.au/tv/(?:game-replays|highlights|condensed-games)/(?P[0-9]+)' # /(?P.+)' + _TESTS = [ + { + 'url': 'https://nbl.com.au/tv/highlights/1310086/Perth-Wildcats-vs.-Sydney-Kings---Game-Highlights', + 'md5': '684a1126879371c3137850b8474ae3c8', + 'info_dict': { + 'id': '1310086', + 'ext': 'mp4', + 'title': 'Perth Wildcats vs. Sydney Kings - Game Highlights', + 'description': 'Watch the Game Highlights from Perth Wildcats vs. Sydney Kings, 03/26/2022', + 'upload_date': '20220326' + } + }, + { + 'url': 'https://nbl.com.au/tv/condensed-games/1310087/Perth-Wildcats-vs.-Sydney-Kings---Condensed-Game', + 'md5': '1f9ac9ea04dc4024e50b27593860a782', + 'info_dict': { + 'id': '1310087', + 'ext': 'mp4', + 'title': 'Perth Wildcats vs. Sydney Kings - Condensed Game', + 'description': 'Watch the Condensed Game from Perth Wildcats vs. Sydney Kings, 03/26/2022', + 'upload_date': '20220326' + } + }, + { + 'url': 'https://nbl.com.au/tv/game-replays/1303323/NBL22-Round-17-Replay---Perth-Wildcats-vs-Sydney-Kings', + 'md5': '8505f02156f756e865a1aa80050eb768', + 'expected_warnings': ['unable to extract OpenGraph description.+'], + 'info_dict': { + 'id': '1303323', + 'ext': 'mp4', + 'title': 'NBL22 Round 17 Replay - Perth Wildcats vs Sydney Kings', + 'upload_date': '20220326' + } + } + ] + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + title = self._html_search_regex(r'(.+?)', webpage, 'title') + stream_url = self._download_json( + "https://ott.nbl.com.au/api/v2/content/" + video_id + "/access/hls", + video_id, + data=b"", + headers={ + "Referer": "https://ott.nbl.com.au/en-int/embed/" + video_id, + "Origin": "https://ott.nbl.com.au", + "Content-Length": "0" + } + )["data"]["stream"] + formats = self._extract_m3u8_formats(stream_url, video_id, ext="mp4") + + return { + 'id': video_id, + 'title': title, + 'description': self._og_search_description(webpage, default=None), + 'upload_date': unified_strdate(self._og_search_property('video:release_date', webpage, 'upload_date')), + 'formats': formats + } From 0d1bb1ce255f67d18ef4e53ab1d88303c0a71a91 Mon Sep 17 00:00:00 2001 From: alyssadev Date: Tue, 29 Mar 2022 18:29:04 +1000 Subject: [PATCH 2/4] [nbl] re-enable display_id match for testing --- youtube_dl/extractor/nbl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/nbl.py b/youtube_dl/extractor/nbl.py index 0c3c1e159..da6cf492d 100644 --- a/youtube_dl/extractor/nbl.py +++ b/youtube_dl/extractor/nbl.py @@ -6,7 +6,7 @@ from ..utils import unified_strdate class NBLIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?nbl\.com\.au/tv/(?:game-replays|highlights|condensed-games)/(?P[0-9]+)' # /(?P.+)' + _VALID_URL = r'https?://(?:www\.)?nbl\.com\.au/tv/(?:game-replays|highlights|condensed-games)/(?P[0-9]+)/(?P.+)' _TESTS = [ { 'url': 'https://nbl.com.au/tv/highlights/1310086/Perth-Wildcats-vs.-Sydney-Kings---Game-Highlights', From 0ae2a8f32056ecc3a8fb3c8fc9d36ac33066b7b3 Mon Sep 17 00:00:00 2001 From: alyssadev Date: Fri, 1 Apr 2022 10:11:08 +1000 Subject: [PATCH 3/4] [nbl] addressed comments, added more test cases --- youtube_dl/extractor/nbl.py | 39 ++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/youtube_dl/extractor/nbl.py b/youtube_dl/extractor/nbl.py index da6cf492d..1e0ffe4d6 100644 --- a/youtube_dl/extractor/nbl.py +++ b/youtube_dl/extractor/nbl.py @@ -1,12 +1,14 @@ # coding: utf-8 from __future__ import unicode_literals +import re + from .common import InfoExtractor from ..utils import unified_strdate class NBLIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?nbl\.com\.au/tv/(?:game-replays|highlights|condensed-games)/(?P[0-9]+)/(?P.+)' + _VALID_URL = r'https?://(?:www\.)?nbl\.com\.au/tv/(?:game-replays|highlights|condensed-games)?/?(?P[0-9]+)/?(?P.*)' _TESTS = [ { 'url': 'https://nbl.com.au/tv/highlights/1310086/Perth-Wildcats-vs.-Sydney-Kings---Game-Highlights', @@ -16,6 +18,7 @@ class NBLIE(InfoExtractor): 'ext': 'mp4', 'title': 'Perth Wildcats vs. Sydney Kings - Game Highlights', 'description': 'Watch the Game Highlights from Perth Wildcats vs. Sydney Kings, 03/26/2022', + 'display_id': 'Perth-Wildcats-vs.-Sydney-Kings---Game-Highlights', 'upload_date': '20220326' } }, @@ -27,17 +30,29 @@ class NBLIE(InfoExtractor): 'ext': 'mp4', 'title': 'Perth Wildcats vs. Sydney Kings - Condensed Game', 'description': 'Watch the Condensed Game from Perth Wildcats vs. Sydney Kings, 03/26/2022', + 'display_id': 'Perth-Wildcats-vs.-Sydney-Kings---Condensed-Game', 'upload_date': '20220326' } }, { 'url': 'https://nbl.com.au/tv/game-replays/1303323/NBL22-Round-17-Replay---Perth-Wildcats-vs-Sydney-Kings', 'md5': '8505f02156f756e865a1aa80050eb768', - 'expected_warnings': ['unable to extract OpenGraph description.+'], 'info_dict': { 'id': '1303323', 'ext': 'mp4', 'title': 'NBL22 Round 17 Replay - Perth Wildcats vs Sydney Kings', + 'display_id': 'NBL22-Round-17-Replay---Perth-Wildcats-vs-Sydney-Kings', + 'upload_date': '20220326' + } + }, + { + 'url': 'https://nbl.com.au/tv/game-replays/1303323', + 'md5': '8505f02156f756e865a1aa80050eb768', + 'info_dict': { + 'id': '1303323', + 'ext': 'mp4', + 'title': 'NBL22 Round 17 Replay - Perth Wildcats vs Sydney Kings', + 'display_id': 'NBL22-Round-17-Replay---Perth-Wildcats-vs-Sydney-Kings', 'upload_date': '20220326' } } @@ -45,25 +60,27 @@ class NBLIE(InfoExtractor): def _real_extract(self, url): video_id = self._match_id(url) + mobj = re.match(self._VALID_URL, url) webpage = self._download_webpage(url, video_id) - title = self._html_search_regex(r'(.+?)', webpage, 'title') + title = self._html_search_regex(r']*>\s*(.+?)\s*', webpage, 'title') stream_url = self._download_json( - "https://ott.nbl.com.au/api/v2/content/" + video_id + "/access/hls", + 'https://ott.nbl.com.au/api/v2/content/' + video_id + '/access/hls', video_id, - data=b"", + data=b'', headers={ - "Referer": "https://ott.nbl.com.au/en-int/embed/" + video_id, - "Origin": "https://ott.nbl.com.au", - "Content-Length": "0" + 'Referer': 'https://ott.nbl.com.au/en-int/embed/' + video_id, + 'Origin': 'https://ott.nbl.com.au', + 'Content-Length': '0' } - )["data"]["stream"] - formats = self._extract_m3u8_formats(stream_url, video_id, ext="mp4") + )['data']['stream'] + formats = self._extract_m3u8_formats(stream_url, video_id, ext='mp4') return { 'id': video_id, 'title': title, 'description': self._og_search_description(webpage, default=None), - 'upload_date': unified_strdate(self._og_search_property('video:release_date', webpage, 'upload_date')), + 'display_id': mobj.groupdict().get('display_id') or title.replace(" ", "-"), + 'upload_date': unified_strdate(self._og_search_property('video:release_date', webpage, 'upload_date', fatal=False)), 'formats': formats } From 819a5f30a6d977d7a3e2476e2b5db3f798d262e0 Mon Sep 17 00:00:00 2001 From: alyssadev Date: Fri, 1 Apr 2022 10:32:17 +1000 Subject: [PATCH 4/4] [nbl] added support for ott urls --- youtube_dl/extractor/nbl.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/nbl.py b/youtube_dl/extractor/nbl.py index 1e0ffe4d6..464f40200 100644 --- a/youtube_dl/extractor/nbl.py +++ b/youtube_dl/extractor/nbl.py @@ -8,7 +8,7 @@ from ..utils import unified_strdate class NBLIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?nbl\.com\.au/tv/(?:game-replays|highlights|condensed-games)?/?(?P[0-9]+)/?(?P.*)' + _VALID_URL = r'https?://(?:www|ott\.)?nbl\.com\.au/(?:tv|en(?:-int)?)/(?P
game-replays|highlights|condensed-games|embed)?/?(?P[0-9]+)/?(?P.*)' _TESTS = [ { 'url': 'https://nbl.com.au/tv/highlights/1310086/Perth-Wildcats-vs.-Sydney-Kings---Game-Highlights', @@ -55,13 +55,26 @@ class NBLIE(InfoExtractor): 'display_id': 'NBL22-Round-17-Replay---Perth-Wildcats-vs-Sydney-Kings', 'upload_date': '20220326' } + }, + { + 'url': 'https://ott.nbl.com.au/en-int/embed/1303323', + 'md5': '8505f02156f756e865a1aa80050eb768', + 'info_dict': { + 'id': '1303323', + 'ext': 'mp4', + 'title': 'NBL22 Round 17 Replay - Perth Wildcats vs Sydney Kings', + 'display_id': 'NBL22-Round-17-Replay---Perth-Wildcats-vs-Sydney-Kings' + } } ] def _real_extract(self, url): video_id = self._match_id(url) mobj = re.match(self._VALID_URL, url) - webpage = self._download_webpage(url, video_id) + if mobj.groupdict().get("section") not in ['game-replays', 'highlights', 'condensed-games']: + webpage = self._download_webpage('https://ott.nbl.com.au/en-int/embed/' + video_id, video_id) + else: + webpage = self._download_webpage(url, video_id) title = self._html_search_regex(r']*>\s*(.+?)\s*', webpage, 'title') stream_url = self._download_json( @@ -81,6 +94,6 @@ class NBLIE(InfoExtractor): 'title': title, 'description': self._og_search_description(webpage, default=None), 'display_id': mobj.groupdict().get('display_id') or title.replace(" ", "-"), - 'upload_date': unified_strdate(self._og_search_property('video:release_date', webpage, 'upload_date', fatal=False)), + 'upload_date': unified_strdate(self._og_search_property('video:release_date', webpage, 'upload_date', fatal=False, default=None)), 'formats': formats }