From 272bfb3e477335b7ef116db9b0546e291e1800e4 Mon Sep 17 00:00:00 2001 From: cens0r <74016750+cens0r@users.noreply.github.com> Date: Mon, 18 Apr 2022 19:36:29 +0200 Subject: [PATCH 1/5] Add CamWhoresBay (video / all model videos) --- youtube_dl/extractor/camwhoresbay.py | 143 +++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 youtube_dl/extractor/camwhoresbay.py diff --git a/youtube_dl/extractor/camwhoresbay.py b/youtube_dl/extractor/camwhoresbay.py new file mode 100644 index 000000000..c158a7bfd --- /dev/null +++ b/youtube_dl/extractor/camwhoresbay.py @@ -0,0 +1,143 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor +import re +import itertools +from ..utils import ( + js_to_json, +) + + +class CamWhoresBayVideoIE(InfoExtractor): + IE_NAME = 'camwhoresbay:video' + _VALID_URL = r'https?://(?:www\.)?camwhoresbay\.com/videos/(?P[0-9]+)/.+' + _TEST = { + 'url': 'https://www.camwhoresbay.com/videos/484472/kirsten-xxx-2022-01-19/', + 'md5': 'ae3c5c34866afbd062adf5d2321d66f3', + 'info_dict': { + 'id': '484472', + 'ext': 'mp4', + 'title': 'Kirsten_Xxx 2022-01-19', + 'uploader': '789sani', + 'url': r're:https://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472_720p\.mp4/\?rnd=.+', + # 'formats': [{ + # 'url': r're:https://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472\.mp4/\?rnd=.+', + # 'height': 360 + # }, { + # 'url': r're:https://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472_480p\.mp4/\?rnd=.+', + # 'height': 480 + # }, { + # 'url': r're:https://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472_720p\.mp4/\?rnd=.+', + # 'height': 720 + # }] + 'thumbnail': 'https://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview.mp4.jpg', + 'thumbnails': [{ + 'url': 'https://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview.mp4.jpg', + 'height': 360 + }, { + 'url': 'https://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview_480p.mp4.jpg', + 'height': 480 + }, { + 'url': 'https://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview_720p.mp4.jpg', + 'height': 720 + }], + # TODO more properties, either as: + # * A value + # * MD5 checksum; start the string with md5: + # * A regular expression; start the string with re: + # * Any Python type (for example int or float) + } + } + + @staticmethod + def _parse_formats(flashvars): + formats = [] + rnd = flashvars.get('rnd') + for k, v in flashvars.items(): + if re.match(r"^video_(alt_)?url\d?$", k): + f = { + 'url': '{}?rnd={}'.format(v, rnd), + 'height': int(flashvars['{}_text'.format(k)][:-1]) + } + formats.append(f) + return formats + + @staticmethod + def _parse_thumbnails(flashvars): + thumbnails = [] + for k, v in flashvars.items(): + if re.match(r"^preview_url\d$", k): + t = { + 'url': 'https:{}'.format(v), + 'height': int(flashvars[k.replace('_url', '_height')]) + } + thumbnails.append(t) + return thumbnails + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + title = self._html_search_regex(r'
\n\s*

(.+?)

', webpage, 'title') + + flashvars = self._parse_json( + self._search_regex(r'var\s+flashvars\s+=\s+({\n.+});', webpage, 'flashvars', default='{}'), + video_id, transform_source=js_to_json) + uploader = self._search_regex(r'class=\"avatar\" href=\"http.+/\" title=\"(.+)\"', webpage, 'uploader', + fatal=False) + + formats = self._parse_formats(flashvars) + thumbnails = self._parse_thumbnails(flashvars) + + return { + 'id': video_id, + 'title': title, + 'uploader': uploader, + 'url': formats[0]['url'], + 'formats': formats, + 'thumbnail': thumbnails[0]['url'], + 'thumbnails': thumbnails, + } + + +class CamWhoresBayModelIE(CamWhoresBayVideoIE): + IE_NAME = 'camwhoresbay:model' + _VALID_URL = r'https?://(?:www\.)?camwhoresbay\.com/models/(?P[0-9a-z\-]+)/?' + _MORE_PAGES_INDICATOR = r'\n\s*

\s*(.*?)\'s New Videos.+

' + _BASE_URL_TEMPL = 'https://www.camwhoresbay.com/models/%s/?mode=async&function=get_block&block_id=list_videos_common_videos_list&sort_by=post_date&from=' + _TEST = { + 'url': 'https://www.camwhoresbay.com/models/kirsten-xxx/', + 'info_dict': { + 'id': 'kirsten-xxx', + 'title': 'Kirsten_xxx', + } + } + _PAGE_SIZE = 35 + + def _extract_list_title(self, webpage): + return self._TITLE or self._html_search_regex( + self._TITLE_RE, webpage, 'list title', fatal=False) + + def _title_and_entries(self, model_id, base_url): + for page in itertools.count(1): + webpage = self._download_webpage('%s%d' % (base_url, page), model_id, 'Downloading page %s' % page) + + if page == 1: + yield self._extract_list_title(webpage) + + for video_id in re.findall(r'\n\s* Date: Mon, 18 Apr 2022 19:37:41 +0200 Subject: [PATCH 2/5] Enable CamWhoresBay extractor --- youtube_dl/extractor/extractors.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 535080d0a..e64d09cd1 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -164,6 +164,10 @@ from .camdemy import ( ) from .cammodels import CamModelsIE from .camtube import CamTubeIE +from .camwhoresbay import ( + CamWhoresBayVideoIE, + CamWhoresBayModelIE, +) from .camwithher import CamWithHerIE from .canalplus import CanalplusIE from .canalc2 import Canalc2IE From 31825563c7cf9a12a13446d35a2df884eaa499ba Mon Sep 17 00:00:00 2001 From: cens0r <74016750+cens0r@users.noreply.github.com> Date: Mon, 18 Apr 2022 19:50:27 +0200 Subject: [PATCH 3/5] minor fixes --- youtube_dl/extractor/camwhoresbay.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/youtube_dl/extractor/camwhoresbay.py b/youtube_dl/extractor/camwhoresbay.py index c158a7bfd..2c00e4b2e 100644 --- a/youtube_dl/extractor/camwhoresbay.py +++ b/youtube_dl/extractor/camwhoresbay.py @@ -31,7 +31,6 @@ class CamWhoresBayVideoIE(InfoExtractor): # 'url': r're:https://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472_720p\.mp4/\?rnd=.+', # 'height': 720 # }] - 'thumbnail': 'https://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview.mp4.jpg', 'thumbnails': [{ 'url': 'https://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview.mp4.jpg', 'height': 360 @@ -93,9 +92,8 @@ class CamWhoresBayVideoIE(InfoExtractor): 'id': video_id, 'title': title, 'uploader': uploader, - 'url': formats[0]['url'], + 'url': formats[0].get('url'), 'formats': formats, - 'thumbnail': thumbnails[0]['url'], 'thumbnails': thumbnails, } From a764865324e990d05e4bbf1ec24bb1f11844ac73 Mon Sep 17 00:00:00 2001 From: cens0r <74016750+cens0r@users.noreply.github.com> Date: Sun, 8 May 2022 16:38:40 +0200 Subject: [PATCH 4/5] fixes and suggestions --- youtube_dl/extractor/camwhoresbay.py | 63 +++++++++++----------------- 1 file changed, 25 insertions(+), 38 deletions(-) diff --git a/youtube_dl/extractor/camwhoresbay.py b/youtube_dl/extractor/camwhoresbay.py index 2c00e4b2e..5a432af41 100644 --- a/youtube_dl/extractor/camwhoresbay.py +++ b/youtube_dl/extractor/camwhoresbay.py @@ -5,7 +5,11 @@ from .common import InfoExtractor import re import itertools from ..utils import ( + int_or_none, js_to_json, + parse_resolution, + sanitize_url, + update_url_query, ) @@ -21,31 +25,7 @@ class CamWhoresBayVideoIE(InfoExtractor): 'title': 'Kirsten_Xxx 2022-01-19', 'uploader': '789sani', 'url': r're:https://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472_720p\.mp4/\?rnd=.+', - # 'formats': [{ - # 'url': r're:https://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472\.mp4/\?rnd=.+', - # 'height': 360 - # }, { - # 'url': r're:https://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472_480p\.mp4/\?rnd=.+', - # 'height': 480 - # }, { - # 'url': r're:https://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472_720p\.mp4/\?rnd=.+', - # 'height': 720 - # }] - 'thumbnails': [{ - 'url': 'https://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview.mp4.jpg', - 'height': 360 - }, { - 'url': 'https://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview_480p.mp4.jpg', - 'height': 480 - }, { - 'url': 'https://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview_720p.mp4.jpg', - 'height': 720 - }], - # TODO more properties, either as: - # * A value - # * MD5 checksum; start the string with md5: - # * A regular expression; start the string with re: - # * Any Python type (for example int or float) + 'thumbnail': r're:^https?://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview_720p.mp4.jpg', } } @@ -54,11 +34,11 @@ class CamWhoresBayVideoIE(InfoExtractor): formats = [] rnd = flashvars.get('rnd') for k, v in flashvars.items(): - if re.match(r"^video_(alt_)?url\d?$", k): + if re.match(r'^video_(alt_)?url\d?$', k): f = { - 'url': '{}?rnd={}'.format(v, rnd), - 'height': int(flashvars['{}_text'.format(k)][:-1]) + 'url': update_url_query(v, {'rnd': rnd, }) } + f.update(parse_resolution(flashvars.get(k + '_text'))) formats.append(f) return formats @@ -66,10 +46,10 @@ class CamWhoresBayVideoIE(InfoExtractor): def _parse_thumbnails(flashvars): thumbnails = [] for k, v in flashvars.items(): - if re.match(r"^preview_url\d$", k): + if re.match(r'^preview_url\d$', k): t = { - 'url': 'https:{}'.format(v), - 'height': int(flashvars[k.replace('_url', '_height')]) + 'url': sanitize_url(v), + 'height': int_or_none(flashvars.get(k.replace('_url', '_height'))), } thumbnails.append(t) return thumbnails @@ -77,12 +57,17 @@ class CamWhoresBayVideoIE(InfoExtractor): def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) + + if re.search(r'(?s)
\n\s*

(.+?)

', webpage, 'title') flashvars = self._parse_json( - self._search_regex(r'var\s+flashvars\s+=\s+({\n.+});', webpage, 'flashvars', default='{}'), + self._search_regex(r'(?s)var\s+flashvars\s*=\s*({.+?})\s*;', webpage, 'flashvars', default='{}'), video_id, transform_source=js_to_json) - uploader = self._search_regex(r'class=\"avatar\" href=\"http.+/\" title=\"(.+)\"', webpage, 'uploader', + uploader = self._search_regex(r'''(?s)\bclass\s*=\s*["']avatar["'][^>]+?\btitle\s*=\s*["'](.+)["']''', webpage, 'uploader', fatal=False) formats = self._parse_formats(flashvars) @@ -101,9 +86,9 @@ class CamWhoresBayVideoIE(InfoExtractor): class CamWhoresBayModelIE(CamWhoresBayVideoIE): IE_NAME = 'camwhoresbay:model' _VALID_URL = r'https?://(?:www\.)?camwhoresbay\.com/models/(?P[0-9a-z\-]+)/?' - _MORE_PAGES_INDICATOR = r']*\bclass\s*=\s*["']next["'][^>]*>]*href\s*=\s*["']#videos["']''' _TITLE = None - _TITLE_RE = r'\n\s*

\s*(.*?)\'s New Videos.+

' + _TITLE_RE = r'''(?is)]+?\bclass\s*=\s*["']headline["'][^>]*>\s*]*>\s*(.*?)'s\s+New\s+Videos\b''' _BASE_URL_TEMPL = 'https://www.camwhoresbay.com/models/%s/?mode=async&function=get_block&block_id=list_videos_common_videos_list&sort_by=post_date&from=' _TEST = { 'url': 'https://www.camwhoresbay.com/models/kirsten-xxx/', @@ -115,7 +100,7 @@ class CamWhoresBayModelIE(CamWhoresBayVideoIE): _PAGE_SIZE = 35 def _extract_list_title(self, webpage): - return self._TITLE or self._html_search_regex( + return self._html_search_regex( self._TITLE_RE, webpage, 'list title', fatal=False) def _title_and_entries(self, model_id, base_url): @@ -125,7 +110,9 @@ class CamWhoresBayModelIE(CamWhoresBayVideoIE): if page == 1: yield self._extract_list_title(webpage) - for video_id in re.findall(r'\n\s*]*\bclass\s*=\s*["\'.*?\bvideo\-item\b.*?["\'][^>]*>\s*]*\bhref\s*=\s*["\'](.*?)["\']', + webpage): yield self.url_result(video_id) if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None: @@ -138,4 +125,4 @@ class CamWhoresBayModelIE(CamWhoresBayVideoIE): def _real_extract(self, url): model_id = self._match_id(url) - return self._extract_videos(model_id, self._BASE_URL_TEMPL % model_id) + return self._extract_videos(model_id, self._BASE_URL_TEMPL % (model_id,)) From a7c1a558edb216b49789e189e0e3d5cc8ca46f51 Mon Sep 17 00:00:00 2001 From: cens0r <74016750+cens0r@users.noreply.github.com> Date: Sun, 8 May 2022 16:41:44 +0200 Subject: [PATCH 5/5] Update youtube_dl/extractor/camwhoresbay.py Co-authored-by: dirkf --- youtube_dl/extractor/camwhoresbay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/camwhoresbay.py b/youtube_dl/extractor/camwhoresbay.py index 5a432af41..6d1596644 100644 --- a/youtube_dl/extractor/camwhoresbay.py +++ b/youtube_dl/extractor/camwhoresbay.py @@ -24,7 +24,7 @@ class CamWhoresBayVideoIE(InfoExtractor): 'ext': 'mp4', 'title': 'Kirsten_Xxx 2022-01-19', 'uploader': '789sani', - 'url': r're:https://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472_720p\.mp4/\?rnd=.+', + 'url': r're:^https?://www\.camwhoresbay\.com/get_file/7/55259a27805bf1313318c14b2afb0dae1fef6e1dd4/484000/484472/484472_720p\.mp4/\?rnd=.+', 'thumbnail': r're:^https?://cwbstatic.cdntrex.com/contents/videos_screenshots/484000/484472/preview_720p.mp4.jpg', } }