From 77e8f5353cb519bd5a9e3699d10de2f90ba07c73 Mon Sep 17 00:00:00 2001 From: Zenon Mousmoulas Date: Wed, 10 Nov 2021 22:21:14 +0200 Subject: [PATCH 1/3] JSON-LD: Support top-level @graph expressing implicit default graph MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per W3C JSON-LD v1.1 §4.9 (non-normative ref): When a JSON-LD document's top-level structure is a map that contains no other keys than @graph and optionally @context (properties that are not mapped to an IRI or a keyword are ignored), @graph is considered to express the otherwise implicit default graph. Support such a structure in InfoExtractor._json_ld parsing: Wrap the control flow block in a function, which is called recursively upon such a structure --- youtube_dl/extractor/common.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 797c35fd5..d947b9253 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -1296,8 +1296,16 @@ class InfoExtractor(object): }) extract_interaction_statistic(e) - for e in json_ld: - if '@context' in e: + def traverse_json_ld(json_ld, info, at_top_level=True): + for e in json_ld: + if at_top_level and '@context' not in e: + continue + if at_top_level and all(k in ('@context', '@graph') for k in e): + graph = e['@graph'] + if isinstance(graph, dict): + graph = [graph] + traverse_json_ld(graph, info, at_top_level=False) + break item_type = e.get('@type') if expected_type is not None and expected_type != item_type: continue @@ -1345,6 +1353,8 @@ class InfoExtractor(object): continue else: break + traverse_json_ld(json_ld, info) + return dict((k, v) for k, v in info.items() if v is not None) @staticmethod From d6469de1da454d7afbd66eb45ed06d741a1b9e8d Mon Sep 17 00:00:00 2001 From: Zenon Mousmoulas Date: Fri, 12 Nov 2021 08:49:35 +0200 Subject: [PATCH 2/3] Extend TestInfoExtractor.test_search_json_ld_realworld to cover @graph expressing JSON-LD implicit default graph * Refactor tests in a list of 3-tuples: test html string, expected dict, keyword args for InfoExtractor._search_json_ld * Adapt test code accordingly * Add test for @graph expressing JSON-LD implicit default graph --- test/test_InfoExtractor.py | 99 ++++++++++++++++++++++++++++++++------ 1 file changed, 84 insertions(+), 15 deletions(-) diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py index dd69a681b..063f7f1f2 100644 --- a/test/test_InfoExtractor.py +++ b/test/test_InfoExtractor.py @@ -99,10 +99,10 @@ class TestInfoExtractor(unittest.TestCase): self.assertRaises(RegexNotFoundError, ie._html_search_meta, ('z', 'x'), html, None, fatal=True) def test_search_json_ld_realworld(self): - # https://github.com/ytdl-org/youtube-dl/issues/23306 - expect_dict( - self, - self.ie._search_json_ld(r'''''', None), - { - 'title': '1 On 1 With Kleio', - 'description': 'Kleio Valentien', - 'url': 'https://gvideo.eporner.com/xN49A1cT3eB/xN49A1cT3eB.mp4', - 'timestamp': 1449347075, - 'duration': 743.0, - 'view_count': 1120958, - 'width': 1920, - 'height': 1080, - }) + ''', + { + 'title': '1 On 1 With Kleio', + 'description': 'Kleio Valentien', + 'url': 'https://gvideo.eporner.com/xN49A1cT3eB/xN49A1cT3eB.mp4', + 'timestamp': 1449347075, + 'duration': 743.0, + 'view_count': 1120958, + 'width': 1920, + 'height': 1080, + }, + {}, + ), + ( + r'''''', + { + 'timestamp': 1636523400, + 'title': 'md5:91fe569e952e4d146485740ae927662b', + }, + {'expected_type': 'NewsArticle'}, + ), + ] + for html, expected_dict, search_json_ld_kwargs in _TESTS: + expect_dict( + self, + self.ie._search_json_ld(html, None, **search_json_ld_kwargs), + expected_dict + ) def test_download_json(self): uri = encode_data_uri(b'{"foo": "blah"}', 'application/json') From a2fd63ce22280cef3856d39d795ec91303152ff6 Mon Sep 17 00:00:00 2001 From: Zenon Mousmoulas Date: Fri, 12 Nov 2021 08:53:59 +0200 Subject: [PATCH 3/3] JSON-LD: Tweak (News)Article description extraction Let JSON-LD extract description from articleBody and fall back to description field when processing (News)Article typed nodes --- youtube_dl/extractor/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index d947b9253..0b62e09a1 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -1338,7 +1338,7 @@ class InfoExtractor(object): info.update({ 'timestamp': parse_iso8601(e.get('datePublished')), 'title': unescapeHTML(e.get('headline')), - 'description': unescapeHTML(e.get('articleBody')), + 'description': unescapeHTML(e.get('articleBody') or e.get('description')), }) elif item_type == 'VideoObject': extract_video_object(e)