From 174624aef80a312d5ef8cae0969d8a3be6c57c79 Mon Sep 17 00:00:00 2001 From: wangchangren Date: Sun, 12 Jun 2022 10:58:44 +0800 Subject: [PATCH 1/7] [QingTing] Add new extractor --- youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/qingting.py | 32 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 youtube_dl/extractor/qingting.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 452caeade..7512bef7f 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -975,6 +975,7 @@ from .presstv import PressTVIE from .prosiebensat1 import ProSiebenSat1IE from .puls4 import Puls4IE from .pyvideo import PyvideoIE +from .qingting import QingTingIE from .qqmusic import ( QQMusicIE, QQMusicSingerIE, diff --git a/youtube_dl/extractor/qingting.py b/youtube_dl/extractor/qingting.py new file mode 100644 index 000000000..ffce86408 --- /dev/null +++ b/youtube_dl/extractor/qingting.py @@ -0,0 +1,32 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor + + +class QingTingIE(InfoExtractor): + IE_NAME = 'QingTing' + _VALID_URL = r'(?:https?://)?(?:www\.)?m\.((qingting\.fm)|(qtfm\.cn))/vchannels/\d+/programs/(?P\d+)' + _TEST = { + 'url': 'https://m.qingting.fm/vchannels/378005/programs/22257411/', + 'md5': '47e6a94f4e621ed832c316fd1888fb3c', + 'info_dict': { + 'id': '22257411', + 'ext': 'mp3', + 'title': '用了十年才修改，谁在乎教科书？-睡前消息-蜻蜓FM听头条', + } + } + + def _real_extract(self, url): + video_id = re.search(self._VALID_URL, url).group('id') + webpage = self._download_webpage(url, video_id) + title = self._html_search_regex(r'(.*)', webpage, 'title') or self._og_search_title(webpage) + url = re.search(r'\"audioUrl\"\s*:\s*\"(?P.*?)\"', webpage).group('url') + return { + 'id': video_id, + 'title': title, + 'ext': 'mp3', + 'url': url, + } From 60783025df3928c953341856508dfbc294c1aa1c Mon Sep 17 00:00:00 2001 From: changren-wcr <105254603+changren-wcr@users.noreply.github.com> Date: Thu, 16 Jun 2022 19:02:30 +0800 Subject: [PATCH 2/7] remove capture of patterns that aren't used Co-authored-by: dirkf --- youtube_dl/extractor/qingting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/qingting.py b/youtube_dl/extractor/qingting.py index ffce86408..eed345d38 100644 --- a/youtube_dl/extractor/qingting.py +++ b/youtube_dl/extractor/qingting.py @@ -8,7 +8,7 @@ from .common import InfoExtractor class QingTingIE(InfoExtractor): IE_NAME = 'QingTing' - _VALID_URL = r'(?:https?://)?(?:www\.)?m\.((qingting\.fm)|(qtfm\.cn))/vchannels/\d+/programs/(?P\d+)' + _VALID_URL = r'(?:https?://)?(?:www\.)?m\.(?:qingting\.fm|qtfm\.cn)/vchannels/\d+/programs/(?P\d+)' _TEST = { 'url': 'https://m.qingting.fm/vchannels/378005/programs/22257411/', 'md5': '47e6a94f4e621ed832c316fd1888fb3c', From 87706c5ec86191a3d107159d0380f2058ce1708a Mon Sep 17 00:00:00 2001 From: changren-wcr <105254603+changren-wcr@users.noreply.github.com> Date: Thu, 16 Jun 2022 19:08:09 +0800 Subject: [PATCH 3/7] Use _search_regex() to get proper error reports in youtube-dl Co-authored-by: dirkf --- youtube_dl/extractor/qingting.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/qingting.py b/youtube_dl/extractor/qingting.py index eed345d38..831659303 100644 --- a/youtube_dl/extractor/qingting.py +++ b/youtube_dl/extractor/qingting.py @@ -23,7 +23,13 @@ class QingTingIE(InfoExtractor): video_id = re.search(self._VALID_URL, url).group('id') webpage = self._download_webpage(url, video_id) title = self._html_search_regex(r'(.*)', webpage, 'title') or self._og_search_title(webpage) - url = re.search(r'\"audioUrl\"\s*:\s*\"(?P.*?)\"', webpage).group('url') + url = self._search_regex( + r'''("|')audioUrl\1\s*:\s*("|')(?P(?:(?!\2).)*)\2''', + webpage, 'audio URL') + test_url = url_or_none(url) + if not test_url: + raise ExtractorError('Invalid audio URL %s' % (url, )) + url = test_url return { 'id': video_id, 'title': title, From 9a421b4e7e2ec09c7559f2cd9e3e40164592fde7 Mon Sep 17 00:00:00 2001 From: changren-wcr <105254603+changren-wcr@users.noreply.github.com> Date: Thu, 16 Jun 2022 19:12:26 +0800 Subject: [PATCH 4/7] fix regular search pattern for title _html_search_regex() has default fatal=True: add a default to fall back to _og_search_title() allow line break in .* Co-authored-by: dirkf --- youtube_dl/extractor/qingting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/qingting.py b/youtube_dl/extractor/qingting.py index 831659303..04fabbb37 100644 --- a/youtube_dl/extractor/qingting.py +++ b/youtube_dl/extractor/qingting.py @@ -22,7 +22,7 @@ class QingTingIE(InfoExtractor): def _real_extract(self, url): video_id = re.search(self._VALID_URL, url).group('id') webpage = self._download_webpage(url, video_id) - title = self._html_search_regex(r'(.*)', webpage, 'title') or self._og_search_title(webpage) + title = self._html_search_regex(r'(?s)]*>(.*)', webpage, 'title', default=None) or self._og_search_title(webpage) url = self._search_regex( r'''("|')audioUrl\1\s*:\s*("|')(?P(?:(?!\2).)*)\2''', webpage, 'audio URL') From 16e7b15f76dc4d8755a8c4091b23ffdd57a89c59 Mon Sep 17 00:00:00 2001 From: changren-wcr <105254603+changren-wcr@users.noreply.github.com> Date: Thu, 16 Jun 2022 19:14:08 +0800 Subject: [PATCH 5/7] use youtube-dl match function instead of native python re Co-authored-by: dirkf --- youtube_dl/extractor/qingting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/qingting.py b/youtube_dl/extractor/qingting.py index 04fabbb37..72a44e3e6 100644 --- a/youtube_dl/extractor/qingting.py +++ b/youtube_dl/extractor/qingting.py @@ -20,7 +20,7 @@ class QingTingIE(InfoExtractor): } def _real_extract(self, url): - video_id = re.search(self._VALID_URL, url).group('id') + video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) title = self._html_search_regex(r'(?s)]*>(.*)', webpage, 'title', default=None) or self._og_search_title(webpage) url = self._search_regex( From 9ff181a1ab649ee7903d97ffefbf881cd0491f4e Mon Sep 17 00:00:00 2001 From: wangchangren Date: Thu, 16 Jun 2022 20:43:37 +0800 Subject: [PATCH 6/7] fix: add new extractor for QingTing --- youtube_dl/extractor/extractors.py | 5 ++++- youtube_dl/extractor/qingting.py | 36 ++++++++++++++++++++++++++---- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 7512bef7f..9bb23ecc4 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -975,7 +975,10 @@ from .presstv import PressTVIE from .prosiebensat1 import ProSiebenSat1IE from .puls4 import Puls4IE from .pyvideo import PyvideoIE -from .qingting import QingTingIE +from .qingting import ( + QingTingMobileIE, + QingTingDeskTopIE, +) from .qqmusic import ( QQMusicIE, QQMusicSingerIE, diff --git a/youtube_dl/extractor/qingting.py b/youtube_dl/extractor/qingting.py index 72a44e3e6..86a59d3ba 100644 --- a/youtube_dl/extractor/qingting.py +++ b/youtube_dl/extractor/qingting.py @@ -5,8 +5,9 @@ import re from .common import InfoExtractor +from youtube_dl import utils -class QingTingIE(InfoExtractor): +class QingTingMobileIE(InfoExtractor): IE_NAME = 'QingTing' _VALID_URL = r'(?:https?://)?(?:www\.)?m\.(?:qingting\.fm|qtfm\.cn)/vchannels/\d+/programs/(?P\d+)' _TEST = { @@ -25,10 +26,10 @@ class QingTingIE(InfoExtractor): title = self._html_search_regex(r'(?s)]*>(.*)', webpage, 'title', default=None) or self._og_search_title(webpage) url = self._search_regex( r'''("|')audioUrl\1\s*:\s*("|')(?P(?:(?!\2).)*)\2''', - webpage, 'audio URL') - test_url = url_or_none(url) + webpage, 'audio URL', group="url") + test_url = utils.url_or_none(url) if not test_url: - raise ExtractorError('Invalid audio URL %s' % (url, )) + raise utils.ExtractorError('Invalid audio URL %s' % (url,)) url = test_url return { 'id': video_id, @@ -36,3 +37,30 @@ class QingTingIE(InfoExtractor): 'ext': 'mp3', 'url': url, } + +class QingTingDeskTopIE(InfoExtractor): + IE_NAME = 'QingTing' + _VALID_URL = r'(?:https?://)?(?:www\.)?(?:qingting\.fm|qtfm\.cn)/channels/\d+/programs/(?P\d+)' + _TEST = { + 'url': 'https://www.qingting.fm/channels/378005/programs/22257411/', + 'md5': '47e6a94f4e621ed832c316fd1888fb3c', + 'info_dict': { + 'id': '22257411', + 'ext': 'mp3', + 'title': '用了十年才修改，谁在乎教科书？-睡前消息-蜻蜓FM听头条', + } + } + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + title = self._html_search_regex(r'(?s)]*>(.*)', webpage, 'title', + default=None) or self._og_search_title(webpage) + url = self._search_regex( + r'''("|')alternate\1\s*:\s*("|')(?P(?:(?!\2).)*)\2''', + webpage, 'alternate URL', group="url") + test_url = utils.url_or_none(url) + if not test_url: + raise utils.ExtractorError('Invalid audio URL %s' % (url,)) + url = test_url + return self.url_result(url=url, video_id=video_id, video_title=title) From 0afddd0b9ae23e32d809e2f4edd5574e28d28487 Mon Sep 17 00:00:00 2001 From: wangchangren Date: Sat, 18 Jun 2022 08:56:44 +0800 Subject: [PATCH 7/7] fix: refactor two extractors into one --- youtube_dl/extractor/extractors.py | 5 +-- youtube_dl/extractor/qingting.py | 71 ++++++++++++------------------ 2 files changed, 30 insertions(+), 46 deletions(-) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 9bb23ecc4..7512bef7f 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -975,10 +975,7 @@ from .presstv import PressTVIE from .prosiebensat1 import ProSiebenSat1IE from .puls4 import Puls4IE from .pyvideo import PyvideoIE -from .qingting import ( - QingTingMobileIE, - QingTingDeskTopIE, -) +from .qingting import QingTingIE from .qqmusic import ( QQMusicIE, QQMusicSingerIE, diff --git a/youtube_dl/extractor/qingting.py b/youtube_dl/extractor/qingting.py index 86a59d3ba..65a49904d 100644 --- a/youtube_dl/extractor/qingting.py +++ b/youtube_dl/extractor/qingting.py @@ -1,46 +1,17 @@ # coding: utf-8 from __future__ import unicode_literals -import re - from .common import InfoExtractor from youtube_dl import utils -class QingTingMobileIE(InfoExtractor): - IE_NAME = 'QingTing' - _VALID_URL = r'(?:https?://)?(?:www\.)?m\.(?:qingting\.fm|qtfm\.cn)/vchannels/\d+/programs/(?P\d+)' - _TEST = { - 'url': 'https://m.qingting.fm/vchannels/378005/programs/22257411/', - 'md5': '47e6a94f4e621ed832c316fd1888fb3c', - 'info_dict': { - 'id': '22257411', - 'ext': 'mp3', - 'title': '用了十年才修改，谁在乎教科书？-睡前消息-蜻蜓FM听头条', - } - } - def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - title = self._html_search_regex(r'(?s)]*>(.*)', webpage, 'title', default=None) or self._og_search_title(webpage) - url = self._search_regex( - r'''("|')audioUrl\1\s*:\s*("|')(?P(?:(?!\2).)*)\2''', - webpage, 'audio URL', group="url") - test_url = utils.url_or_none(url) - if not test_url: - raise utils.ExtractorError('Invalid audio URL %s' % (url,)) - url = test_url - return { - 'id': video_id, - 'title': title, - 'ext': 'mp3', - 'url': url, - } - -class QingTingDeskTopIE(InfoExtractor): +class QingTingIE(InfoExtractor): IE_NAME = 'QingTing' - _VALID_URL = r'(?:https?://)?(?:www\.)?(?:qingting\.fm|qtfm\.cn)/channels/\d+/programs/(?P\d+)' + _VALID_URL = r'''(?x) + (?:https?://)?(?:www\.)? + (?Pm\.)?(?:qingting\.fm|qtfm\.cn)/(?(m)v|) + channels/\d+/programs/(?P\d+)''' _TEST = { 'url': 'https://www.qingting.fm/channels/378005/programs/22257411/', 'md5': '47e6a94f4e621ed832c316fd1888fb3c', @@ -56,11 +27,27 @@ class QingTingDeskTopIE(InfoExtractor): webpage = self._download_webpage(url, video_id) title = self._html_search_regex(r'(?s)]*>(.*)', webpage, 'title', default=None) or self._og_search_title(webpage) - url = self._search_regex( - r'''("|')alternate\1\s*:\s*("|')(?P(?:(?!\2).)*)\2''', - webpage, 'alternate URL', group="url") - test_url = utils.url_or_none(url) - if not test_url: - raise utils.ExtractorError('Invalid audio URL %s' % (url,)) - url = test_url - return self.url_result(url=url, video_id=video_id, video_title=title) + urlType = self._search_regex( + self._VALID_URL, + url, 'audio URL', group="m") + if urlType == 'm.': + url = self._search_regex( + r'''("|')audioUrl\1\s*:\s*("|')(?P(?:(?!\2).)*)\2''', + webpage, 'audio URL', group="url") + test_url = utils.url_or_none(url) + if not test_url: + raise utils.ExtractorError('Invalid audio URL %s' % (url,)) + return { + 'id': video_id, + 'title': title, + 'ext': 'mp3', + 'url': test_url, + } + else: + url = self._search_regex( + r'''("|')alternate\1\s*:\s*("|')(?P(?:(?!\2).)*)\2''', + webpage, 'alternate URL', group="url") + test_url = utils.url_or_none(url) + if not test_url: + raise utils.ExtractorError('Invalid audio URL %s' % (url,)) + return self.url_result(url=test_url, video_id=video_id, video_title=title)