From 7561727db6490308205d487a3609e2581e9bced3 Mon Sep 17 00:00:00 2001 From: marieell Date: Thu, 10 Feb 2022 21:49:36 +0100 Subject: [PATCH 1/2] Remove extractors for dead pages They all redirect to youporn.com. --- youtube_dl/extractor/extractors.py | 6 -- youtube_dl/extractor/extremetube.py | 50 ----------- youtube_dl/extractor/generic.py | 6 -- youtube_dl/extractor/keezmovies.py | 133 ---------------------------- youtube_dl/extractor/mofosex.py | 79 ----------------- youtube_dl/extractor/tube8.py | 60 ++++++++++++- 6 files changed, 58 insertions(+), 276 deletions(-) delete mode 100644 youtube_dl/extractor/extremetube.py delete mode 100644 youtube_dl/extractor/keezmovies.py delete mode 100644 youtube_dl/extractor/mofosex.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 50b7cb4a0..88c9b0756 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -356,7 +356,6 @@ from .esri import EsriVideoIE from .europa import EuropaIE from .expotv import ExpoTVIE from .expressen import ExpressenIE -from .extremetube import ExtremeTubeIE from .eyedotv import EyedoTVIE from .facebook import ( FacebookIE, @@ -537,7 +536,6 @@ from .kaltura import KalturaIE from .kankan import KankanIE from .karaoketv import KaraoketvIE from .karrierevideos import KarriereVideosIE -from .keezmovies import KeezMoviesIE from .ketnet import KetnetIE from .khanacademy import ( KhanAcademyIE, @@ -686,10 +684,6 @@ from .mlb import ( ) from .mnet import MnetIE from .moevideo import MoeVideoIE -from .mofosex import ( - MofosexIE, - MofosexEmbedIE, -) from .mojvideo import MojvideoIE from .morningstar import MorningstarIE from .motherless import ( diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py deleted file mode 100644 index acd4090fa..000000000 --- a/youtube_dl/extractor/extremetube.py +++ /dev/null @@ -1,50 +0,0 @@ -from __future__ import unicode_literals - -from ..utils import str_to_int -from .keezmovies import KeezMoviesIE - - -class ExtremeTubeIE(KeezMoviesIE): - _VALID_URL = r'https?://(?:www\.)?extremetube\.com/(?:[^/]+/)?video/(?P[^/#?&]+)' - _TESTS = [{ - 'url': 'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431', - 'md5': '92feaafa4b58e82f261e5419f39c60cb', - 'info_dict': { - 'id': 'music-video-14-british-euro-brit-european-cumshots-swallow-652431', - 'ext': 'mp4', - 'title': 'Music Video 14 british euro brit european cumshots swallow', - 'uploader': 'anonim', - 'view_count': int, - 'age_limit': 18, - } - }, { - 'url': 'http://www.extremetube.com/gay/video/abcde-1234', - 'only_matching': True, - }, { - 'url': 'http://www.extremetube.com/video/latina-slut-fucked-by-fat-black-dick', - 'only_matching': True, - }, { - 'url': 'http://www.extremetube.com/video/652431', - 'only_matching': True, - }] - - def _real_extract(self, url): - webpage, info = self._extract_info(url) - - if not info['title']: - info['title'] = self._search_regex( - r']+title="([^"]+)"[^>]*>', webpage, 'title') - - uploader = self._html_search_regex( - r'Uploaded by:\s*]+>\s*]+>(.+?)', - webpage, 'uploader', fatal=False) - view_count = str_to_int(self._search_regex( - r'Views:\s*]+>\s*<[^>]+>([\d,\.]+)[^/]+)-)?(?P\d+)' - _TESTS = [{ - 'url': 'https://www.keezmovies.com/video/arab-wife-want-it-so-bad-i-see-she-thirsty-and-has-tiny-money-18070681', - 'md5': '2ac69cdb882055f71d82db4311732a1a', - 'info_dict': { - 'id': '18070681', - 'display_id': 'arab-wife-want-it-so-bad-i-see-she-thirsty-and-has-tiny-money', - 'ext': 'mp4', - 'title': 'Arab wife want it so bad I see she thirsty and has tiny money.', - 'thumbnail': None, - 'view_count': int, - 'age_limit': 18, - } - }, { - 'url': 'http://www.keezmovies.com/video/18070681', - 'only_matching': True, - }] - - def _extract_info(self, url, fatal=True): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group('id') - display_id = (mobj.group('display_id') - if 'display_id' in mobj.groupdict() - else None) or mobj.group('id') - - webpage = self._download_webpage( - url, display_id, headers={'Cookie': 'age_verified=1'}) - - formats = [] - format_urls = set() - - title = None - thumbnail = None - duration = None - encrypted = False - - def extract_format(format_url, height=None): - format_url = url_or_none(format_url) - if not format_url or not format_url.startswith(('http', '//')): - return - if format_url in format_urls: - return - format_urls.add(format_url) - tbr = int_or_none(self._search_regex( - r'[/_](\d+)[kK][/_]', format_url, 'tbr', default=None)) - if not height: - height = int_or_none(self._search_regex( - r'[/_](\d+)[pP][/_]', format_url, 'height', default=None)) - if encrypted: - format_url = aes_decrypt_text( - video_url, title, 32).decode('utf-8') - formats.append({ - 'url': format_url, - 'format_id': '%dp' % height if height else None, - 'height': height, - 'tbr': tbr, - }) - - flashvars = self._parse_json( - self._search_regex( - r'flashvars\s*=\s*({.+?});', webpage, - 'flashvars', default='{}'), - display_id, fatal=False) - - if flashvars: - title = flashvars.get('video_title') - thumbnail = flashvars.get('image_url') - duration = int_or_none(flashvars.get('video_duration')) - encrypted = flashvars.get('encrypted') is True - for key, value in flashvars.items(): - mobj = re.search(r'quality_(\d+)[pP]', key) - if mobj: - extract_format(value, int(mobj.group(1))) - video_url = flashvars.get('video_url') - if video_url and determine_ext(video_url, None): - extract_format(video_url) - - video_url = self._html_search_regex( - r'flashvars\.video_url\s*=\s*(["\'])(?Phttp.+?)\1', - webpage, 'video url', default=None, group='url') - if video_url: - extract_format(compat_urllib_parse_unquote(video_url)) - - if not formats: - if 'title="This video is no longer available"' in webpage: - raise ExtractorError( - 'Video %s is no longer available' % video_id, expected=True) - - try: - self._sort_formats(formats) - except ExtractorError: - if fatal: - raise - - if not title: - title = self._html_search_regex( - r']*>([^<]+)', webpage, 'title') - - return webpage, { - 'id': video_id, - 'display_id': display_id, - 'title': strip_or_none(title), - 'thumbnail': thumbnail, - 'duration': duration, - 'age_limit': 18, - 'formats': formats, - } - - def _real_extract(self, url): - webpage, info = self._extract_info(url, fatal=False) - if not info['formats']: - return self.url_result(url, 'Generic') - info['view_count'] = str_to_int(self._search_regex( - r'([\d,.]+) Views?', webpage, 'view count', fatal=False)) - return info diff --git a/youtube_dl/extractor/mofosex.py b/youtube_dl/extractor/mofosex.py deleted file mode 100644 index 5234cac02..000000000 --- a/youtube_dl/extractor/mofosex.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import unicode_literals - -import re - -from .common import InfoExtractor -from ..utils import ( - int_or_none, - str_to_int, - unified_strdate, -) -from .keezmovies import KeezMoviesIE - - -class MofosexIE(KeezMoviesIE): - _VALID_URL = r'https?://(?:www\.)?mofosex\.com/videos/(?P\d+)/(?P[^/?#&.]+)\.html' - _TESTS = [{ - 'url': 'http://www.mofosex.com/videos/318131/amateur-teen-playing-and-masturbating-318131.html', - 'md5': '558fcdafbb63a87c019218d6e49daf8a', - 'info_dict': { - 'id': '318131', - 'display_id': 'amateur-teen-playing-and-masturbating-318131', - 'ext': 'mp4', - 'title': 'amateur teen playing and masturbating', - 'thumbnail': r're:^https?://.*\.jpg$', - 'upload_date': '20121114', - 'view_count': int, - 'like_count': int, - 'dislike_count': int, - 'age_limit': 18, - } - }, { - # This video is no longer available - 'url': 'http://www.mofosex.com/videos/5018/japanese-teen-music-video.html', - 'only_matching': True, - }] - - def _real_extract(self, url): - webpage, info = self._extract_info(url) - - view_count = str_to_int(self._search_regex( - r'VIEWS:\s*([\d,.]+)', webpage, 'view count', fatal=False)) - like_count = int_or_none(self._search_regex( - r'id=["\']amountLikes["\'][^>]*>(\d+)', webpage, - 'like count', fatal=False)) - dislike_count = int_or_none(self._search_regex( - r'id=["\']amountDislikes["\'][^>]*>(\d+)', webpage, - 'like count', fatal=False)) - upload_date = unified_strdate(self._html_search_regex( - r'Added:([^<]+)', webpage, 'upload date', fatal=False)) - - info.update({ - 'view_count': view_count, - 'like_count': like_count, - 'dislike_count': dislike_count, - 'upload_date': upload_date, - 'thumbnail': self._og_search_thumbnail(webpage), - }) - - return info - - -class MofosexEmbedIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?mofosex\.com/embed/?\?.*?\bvideoid=(?P\d+)' - _TESTS = [{ - 'url': 'https://www.mofosex.com/embed/?videoid=318131&referrer=KM', - 'only_matching': True, - }] - - @staticmethod - def _extract_urls(webpage): - return re.findall( - r']+\bsrc=["\']((?:https?:)?//(?:www\.)?mofosex\.com/embed/?\?.*?\bvideoid=\d+)', - webpage) - - def _real_extract(self, url): - video_id = self._match_id(url) - return self.url_result( - 'http://www.mofosex.com/videos/{0}/{0}.html'.format(video_id), - ie=MofosexIE.ie_key(), video_id=video_id) diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py index db93b0182..0c22fae36 100644 --- a/youtube_dl/extractor/tube8.py +++ b/youtube_dl/extractor/tube8.py @@ -3,13 +3,19 @@ from __future__ import unicode_literals import re from ..utils import ( + determine_ext, + ExtractorError, int_or_none, str_to_int, + strip_or_none, + url_or_none, ) -from .keezmovies import KeezMoviesIE +from .common import InfoExtractor +from ..aes import aes_decrypt_text +from ..compat import compat_urllib_parse_unquote -class Tube8IE(KeezMoviesIE): +class Tube8IE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?tube8\.com/(?:[^/]+/)+(?P[^/]+)/(?P\d+)' _TESTS = [{ 'url': 'http://www.tube8.com/teen/kasia-music-video/229795/', @@ -37,6 +43,56 @@ class Tube8IE(KeezMoviesIE): r']+\bsrc=["\']((?:https?:)?//(?:www\.)?tube8\.com/embed/(?:[^/]+/)+\d+)', webpage) + def _extract_info(self, url, fatal=True): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + display_id = (mobj.group('display_id') + if 'display_id' in mobj.groupdict() + else None) or mobj.group('id') + + webpage = self._download_webpage( + url, display_id, headers={'Cookie': 'age_verified=1'}) + + formats = [] + format_urls = set() + + title = None + thumbnail = None + duration = None + encrypted = False + + def extract_format(format_url, height=None): + format_url = url_or_none(format_url) + if not format_url or not format_url.startswith(('http', '//')): + return + if format_url in format_urls: + return + format_urls.add(format_url) + tbr = int_or_none(self._search_regex( + r'[/_](\d+)[kK][/_]', format_url, 'tbr', default=None)) + if not height: + height = int_or_none(self._search_regex( + r'[/_](\d+)[pP][/_]', format_url, 'height', default=None)) + if encrypted: + format_url = aes_decrypt_text( + video_url, title, 32).decode('utf-8') + formats.append({ + 'url': format_url, + 'format_id': '%dp' % height if height else None, + 'height': height, + 'tbr': tbr, + }) + + flashvars = self._parse_json( + self._search_regex( + r'flashvars\s*=\s*({.+?});', webpage, + 'flashvars', default='{}'), + display_id, fatal=False) + + if flashvars: + title = flashvars.get('video_title') + thumbnail = flashvars.get('image_url') + def _real_extract(self, url): webpage, info = self._extract_info(url) From 48d0994b5b3ef6714dba9510656808883e40df40 Mon Sep 17 00:00:00 2001 From: marieell Date: Thu, 10 Feb 2022 22:27:12 +0100 Subject: [PATCH 2/2] [tube8] Fix extractor a bit, replace broken test --- youtube_dl/extractor/tube8.py | 93 +++++++++++++++++++++-------------- 1 file changed, 55 insertions(+), 38 deletions(-) diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py index 0c22fae36..ad234ffb3 100644 --- a/youtube_dl/extractor/tube8.py +++ b/youtube_dl/extractor/tube8.py @@ -7,7 +7,6 @@ from ..utils import ( ExtractorError, int_or_none, str_to_int, - strip_or_none, url_or_none, ) from .common import InfoExtractor @@ -18,23 +17,19 @@ from ..compat import compat_urllib_parse_unquote class Tube8IE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?tube8\.com/(?:[^/]+/)+(?P[^/]+)/(?P\d+)' _TESTS = [{ - 'url': 'http://www.tube8.com/teen/kasia-music-video/229795/', - 'md5': '65e20c48e6abff62ed0c3965fff13a39', + 'url': 'https://www.tube8.com/erotic/playtime/81807731/', + 'md5': 'fefa69ff76debaa63aa59374bfc51c95', 'info_dict': { - 'id': '229795', - 'display_id': 'kasia-music-video', + 'id': '81807731', + 'display_id': 'playtime', 'ext': 'mp4', - 'description': 'hot teen Kasia grinding', - 'uploader': 'unknown', - 'title': 'Kasia music video', + 'uploader': 'kikkijay-ph', + 'title': 'Playtime', 'age_limit': 18, - 'duration': 230, - 'categories': ['Teen'], - 'tags': ['dancing'], + 'duration': 988, + 'categories': ['Erotic'], + 'tags': ['adult toys', 'big boobs', 'butt', 'masturbate'], }, - }, { - 'url': 'http://www.tube8.com/shemale/teen/blonde-cd-gets-kidnapped-by-two-blacks-and-punished-for-being-a-slutty-girl/19569151/', - 'only_matching': True, }] @staticmethod @@ -43,12 +38,10 @@ class Tube8IE(InfoExtractor): r']+\bsrc=["\']((?:https?:)?//(?:www\.)?tube8\.com/embed/(?:[^/]+/)+\d+)', webpage) - def _extract_info(self, url, fatal=True): + def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('id') - display_id = (mobj.group('display_id') - if 'display_id' in mobj.groupdict() - else None) or mobj.group('id') + display_id = mobj.group('display_id') webpage = self._download_webpage( url, display_id, headers={'Cookie': 'age_verified=1'}) @@ -92,51 +85,75 @@ class Tube8IE(InfoExtractor): if flashvars: title = flashvars.get('video_title') thumbnail = flashvars.get('image_url') + duration = int_or_none(flashvars.get('video_duration')) + encrypted = flashvars.get('encrypted') is True + uploader = flashvars.get('sponsor') + for key, value in flashvars.items(): + mobj = re.search(r'quality_(\d+)[pP]', key) + if mobj: + extract_format(value, int(mobj.group(1))) + video_url = flashvars.get('video_url') + for key, value in flashvars.items(): + mobj = re.search(r'quality_(\d+)[pP]', key) + if mobj: + extract_format(value, int(mobj.group(1))) + video_url = flashvars.get('video_url') + if video_url and determine_ext(video_url, None): + extract_format(video_url) - def _real_extract(self, url): - webpage, info = self._extract_info(url) + video_url = self._html_search_regex( + r'flashvars\.video_url\s*=\s*(["\'])(?Phttp.+?)\1', + webpage, 'video url', default=None, group='url') + if video_url: + extract_format(compat_urllib_parse_unquote(video_url)) - if not info['title']: - info['title'] = self._html_search_regex( - r'videoTitle\s*=\s*"([^"]+)', webpage, 'title') + if not formats: + if 'title="This video is no longer available"' in webpage: + raise ExtractorError( + 'Video %s is no longer available' % video_id, expected=True) - description = self._html_search_regex( - r'(?s)Description:\s*
(.+?)
', webpage, 'description', fatal=False) - uploader = self._html_search_regex( - r'\s*(.+?)\s*<', - webpage, 'uploader', fatal=False) + self._sort_formats(formats) + + if not title: + title = self._html_search_regex([ + r']*>([^<]+)' + r'videoTitle\s*=\s*"([^"]+)'], webpage, 'title', default=display_id.capitalize()) like_count = int_or_none(self._search_regex( - r'rupVar\s*=\s*"(\d+)"', webpage, 'like count', fatal=False)) + r'rupVar\s*=\s*(\d+);', webpage, 'like count', fatal=False)) dislike_count = int_or_none(self._search_regex( - r'rdownVar\s*=\s*"(\d+)"', webpage, 'dislike count', fatal=False)) + r'rdownVar\s*=\s*(\d+);', webpage, 'dislike count', fatal=False)) view_count = str_to_int(self._search_regex( r'Views:\s*\s*
([\d,\.]+)', webpage, 'view count', fatal=False)) comment_count = str_to_int(self._search_regex( - r'(\d+)', + r'\((\d+)\)', webpage, 'comment count', fatal=False)) category = self._search_regex( - r'Category:\s*\s*
\s*]+href=[^>]+>([^<]+)', + r'videoCategoryByName\s*=\s*"([^"]+)";', webpage, 'category', fatal=False) categories = [category] if category else None tags_str = self._search_regex( - r'(?s)Tags:\s*\s*
(.+?)(.*?)", webpage, 'tags', fatal=False) tags = [t for t in re.findall( r']+href=[^>]+>([^<]+)', tags_str)] if tags_str else None - info.update({ - 'description': description, + return { 'uploader': uploader, + 'id': video_id, + 'display_id': display_id, + 'title': title, + 'thumbnail': thumbnail, + 'duration': duration, + 'age_limit': 18, + 'formats': formats, 'view_count': view_count, 'like_count': like_count, 'dislike_count': dislike_count, 'comment_count': comment_count, 'categories': categories, 'tags': tags, - }) - - return info + }